diff --git a/.github/instructions/agent-skills.instructions.md b/.github/instructions/agent-skills.instructions.md index 226977759..60546fa9e 100644 --- a/.github/instructions/agent-skills.instructions.md +++ b/.github/instructions/agent-skills.instructions.md @@ -65,9 +65,32 @@ Before shipping any skill/instruction: `TestOverlaysSkillCoversAllOverlayTypes` extracts the overlay-type enum from the jsonschema tag on `projectconfig.ComponentOverlay.Type` and fails if the skill omits a type. +## Mode-specific content + +azldev has two modes, and the emitted content follows the one it runs in. A `Catalog` +(`NewCatalog(withoutLockfile)`) resolves the skills, instruction wrappers, and templates; +the package-level `Skills`, `Instructions`, `FindSkill`, `SkillDocument`, and `Files` +helpers are the default (lock-file) mode. + +- **Shared by default.** Registry entries and templates under `content/` describe the + default mode and are used by both, so most edits need nothing extra. +- **Replace only what differs.** `withoutLockfileSkills` replaces registry entries by the + name of the default-mode skill it supersedes, and `withoutLockfileInstructions` replaces + instruction descriptions. Pointers to a replaced skill are rewritten automatically. +- **Templates layer.** A template under `content/withoutlockfile/` replaces the + same-named default template for that mode; add one only when the document's content + actually differs. + +When you add or edit a skill, check whether its content names a command that exists in +only one mode (for example `comp update` versus `comp refresh-upstream-commit`) and, if +so, provide the mode-specific variant. Verify both with +`./out/bin/azldev docs agent show --skill ` and +`./out/bin/azldev --without-lockfile docs agent show --skill `. + ## Config-resolved bindings -Repo-specific values (lock dir, rendered-specs dir, work dir) are resolved from the target `azldev.toml` in +Repo-specific values (lock dir, generated upstream-commit dir, rendered-specs dir, work +dir) are resolved from the target `azldev.toml` in [cmds/docs/agent.go](../../internal/app/azldev/cmds/docs/agent.go) and degrade to azldev's defaults when no config is present. To add a binding, extend `Bindings`, resolve it in `resolveBindings`, and reference it in a template as `{{ .FieldName }}`. diff --git a/CHANGELOG.md b/CHANGELOG.md index a222fe318..b3470f419 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,51 @@ All notable changes to `azldev` are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- **`--without-lockfile` preview mode.** Add a global `--without-lockfile` + flag that opts in to a preview of tracking resolved upstream commits in + generated component configuration instead of per-component lock files. The + flag defaults to off; without it azldev's behavior, command set, and + configuration handling are unchanged. The preview surface is not stable and + may change. +- **Generated upstream commit configuration.** With `--without-lockfile`, + record snapshot-selected upstream commits as normal layered TOML under + `base/upstream-commits`. Generated pin files participate in standard + configuration loading, merging, provenance tracking, and validation, and the + project's `lock-dir` setting is accepted but ignored. +- **Upstream commit refresh command.** With `--without-lockfile`, `azldev + component refresh-upstream-commit` resolves and records upstream commits. It + supports check-only operation, removes obsolete pins for selected + non-upstream components, and prunes orphaned generated files when all + components are selected. Configuration is loaded permissively for this + command so stale generated pins can be removed after a component is deleted + or converted to another source type. + +### Changed + +- **Mode-specific component commands.** With `--without-lockfile`, `azldev + component update`, `component history`, and `component query` are replaced by + hidden no-op shims, and the lock-file-only `--skip-lock-validation` flag is + not registered. All of them are unchanged in the default mode. +- **Configuration-based component change detection.** With + `--without-lockfile`, `azldev component changed` loads each historical + project configuration independently and compares normalized build inputs + instead of stored fingerprints. It handles added and deleted components, + resolves recursive includes and inherited defaults at each ref, compares + local source and overlay content, and reports rendered `sources` changes + separately. +- **Synthetic source history.** With `--without-lockfile`, synthetic dist-git + history is built from configured upstream commit transitions and walks + first-parent history to the repository root instead of relying on + lock-recorded import commits. +- **Component workflow guidance.** Agent skills, instruction files, and MCP + tools describe the workflow of the mode azldev runs in. The generated CLI + reference continues to document the default mode; the preview mode is + documented in the user guide. + ## [0.3.0] - 2026-08-10 ### Added diff --git a/docs/user/README.md b/docs/user/README.md index 7db23b199..95fbf8e2c 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -7,6 +7,7 @@ - [Build a Component](./how-to/build-component.md) — build RPMs from component definitions - [Build an Image](./how-to/build-image.md) — build and boot Azure Linux images - [Set Up AI Coding Agents](./how-to/set-up-ai-agents.md) — emit agent skill and instruction files +- [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — opt in to `--without-lockfile` ## Explanation diff --git a/docs/user/explanation/config-system.md b/docs/user/explanation/config-system.md index 721e90632..07c5eca34 100644 --- a/docs/user/explanation/config-system.md +++ b/docs/user/explanation/config-system.md @@ -86,6 +86,8 @@ Component definitions are merged additively. If the same component name (e.g., ` > **Note:** Slice fields (like `overlays`) are **appended**, not replaced, following the same merge behavior used by component configuration inheritance. +> **Preview:** With the global `--without-lockfile` flag, component definitions merge with override semantics instead, and component validation runs only after every included file has been merged, so an individual file may hold a partial component definition. See [Preview the Lock-File-Free Mode](../how-to/preview-without-lockfile.md). + ### Component Groups and Images These are strict-union maps: each name may appear in exactly one config file across the entire include tree. If two files both define `[component-groups.my-group]` or `[images.my-image]`, azldev reports an error. This prevents accidental shadowing and makes it clear where each definition lives. diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md new file mode 100644 index 000000000..b3482e91c --- /dev/null +++ b/docs/user/how-to/preview-without-lockfile.md @@ -0,0 +1,109 @@ +# How To: Preview the Lock-File-Free Mode + +`--without-lockfile` is a **preview** global flag. It selects an alternative way of +tracking a component's resolved upstream commit: instead of per-component lock +files, azldev records the commit in generated component TOML that the project +includes like any other config file. + +The flag is opt-in and defaults to off. Without it, azldev behaves exactly as it +always has — lock files, `component update`, `component history`, and +`component query` are unchanged. Nothing in the preview mode is stable yet; both +the command surface and the generated file layout may change. + +```bash +# Default behavior: lock files. +azldev component render -p curl + +# Preview behavior: generated upstream-commit config. +azldev --without-lockfile component render -p curl +``` + +Pass the flag on every invocation that should use the preview mode, before the +command name. `--without-lockfile=false` explicitly selects the default mode. + +## What Changes + +| Area | Default | `--without-lockfile` | +|------|---------|----------------------| +| Resolved commit storage | `locks/.lock` | `base/upstream-commits/.toml` | +| Refresh command | `azldev component update` | `azldev component refresh-upstream-commit` | +| Inspecting resolved state | `azldev component history`, `azldev component query` | read the generated TOML; no equivalent commands | +| Lock consistency checks | On, with `--skip-lock-validation` to opt out | Not applicable; the flag is not registered | +| `component changed` | Compares stored input fingerprints | Compares project configuration resolved at each ref | +| Synthetic dist-git history | Derived from lock-file fingerprint changes | Derived from generated upstream-commit TOML changes | +| Agent skills and MCP tools | Describe the lock-file workflow | Describe the upstream-commit workflow | + +`component update`, `component history`, and `component query` remain registered +in preview mode as hidden no-ops so that existing scripts report clearly that the +commands do nothing, rather than failing with "unknown command". + +## Configure the Project + +Include the generated directory **before** the component-specific TOML, so that a +component definition can still override the generated pin: + +```toml +includes = [ + "base/upstream-commits/*.toml", + "base/components/*.toml", +] +``` + +Generated files hold only `spec.upstream-commit`; the component's own TOML +supplies the source type and everything else. Because a single file may hold a +partial component definition in this mode, component validation runs after all +config files have been merged. + +An existing `[project] lock-dir` setting is accepted and ignored in preview mode, +so the same project config works in both modes. + +## Refresh a Component + +```bash +# Resolve and record the upstream commit for one component. +azldev --without-lockfile component refresh-upstream-commit -p curl + +# Refresh everything and prune generated files for components that no longer exist. +azldev --without-lockfile component refresh-upstream-commit -a + +# CI gate: exit 1 when any generated file is out of date. +azldev --without-lockfile component refresh-upstream-commit -a --check-only -q +``` + +Refresh after changing a commit pin, upstream distro or version, or snapshot. +Overlay, build-config, and metadata changes do not affect the resolved commit, so +they need only a re-render. + +Commit the refreshed TOML together with the rendered output: synthetic dist-git +history — and therefore `%autorelease` and `%autochangelog` expansion — is derived +from committed changes to the generated file. Unlike the default mode, there is no +fingerprint to compare the working tree against, so uncommitted changes do not +produce a synthetic commit. + +## Detect Changed Components + +```bash +azldev --without-lockfile component changed --from main -a -q -O json +``` + +In preview mode this loads the project configuration independently at both refs +and compares the resolved component build inputs: normalized component +configuration, upstream commit or local spec-directory contents, overlay source +filenames and contents, and the effective distro release version. Documentation, +publishing, test-selection, scheduling-hint, snapshot-time, and checkout-path-only +fields do not mark a component as changed. + +## Emit Agent Files for the Preview Mode + +`azldev docs agent install` emits the content for the mode it runs in, so pass the +flag when the target repository uses the preview workflow: + +```bash +azldev --without-lockfile docs agent install +``` + +## Reference Documentation + +The generated CLI reference under [reference/cli/](../reference/cli/azldev.md) +documents azldev's default mode. Use `azldev --without-lockfile --help` +to see the preview mode's command surface and help text. diff --git a/docs/user/reference/cli/azldev.md b/docs/user/reference/cli/azldev.md index a9072dc0a..8116c782c 100644 --- a/docs/user/reference/cli/azldev.md +++ b/docs/user/reference/cli/azldev.md @@ -29,6 +29,7 @@ lives), or use -C to point to one. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced.md b/docs/user/reference/cli/azldev_advanced.md index 85dc88d00..5cb29e12d 100644 --- a/docs/user/reference/cli/azldev_advanced.md +++ b/docs/user/reference/cli/azldev_advanced.md @@ -32,6 +32,7 @@ output but fully supported. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_download-sources.md b/docs/user/reference/cli/azldev_advanced_download-sources.md index 77102b291..66c7bfd34 100644 --- a/docs/user/reference/cli/azldev_advanced_download-sources.md +++ b/docs/user/reference/cli/azldev_advanced_download-sources.md @@ -70,6 +70,7 @@ azldev advanced download-sources [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mcp.md b/docs/user/reference/cli/azldev_advanced_mcp.md index 1eeef05fe..861981896 100644 --- a/docs/user/reference/cli/azldev_advanced_mcp.md +++ b/docs/user/reference/cli/azldev_advanced_mcp.md @@ -43,6 +43,7 @@ azldev advanced mcp [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock.md b/docs/user/reference/cli/azldev_advanced_mock.md index 3b4a57e63..c7f694261 100644 --- a/docs/user/reference/cli/azldev_advanced_mock.md +++ b/docs/user/reference/cli/azldev_advanced_mock.md @@ -31,6 +31,7 @@ starting interactive shell sessions in mock chroot environments. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md b/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md index b59886f4c..df76d2155 100644 --- a/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md +++ b/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md @@ -50,6 +50,7 @@ azldev advanced mock build-rpms [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock_shell.md b/docs/user/reference/cli/azldev_advanced_mock_shell.md index 9a34a1550..6fbc6e521 100644 --- a/docs/user/reference/cli/azldev_advanced_mock_shell.md +++ b/docs/user/reference/cli/azldev_advanced_mock_shell.md @@ -55,6 +55,7 @@ azldev advanced mock shell [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_wget.md b/docs/user/reference/cli/azldev_advanced_wget.md index 09a159cea..6aa762683 100644 --- a/docs/user/reference/cli/azldev_advanced_wget.md +++ b/docs/user/reference/cli/azldev_advanced_wget.md @@ -44,6 +44,7 @@ azldev advanced wget URI [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion.md b/docs/user/reference/cli/azldev_completion.md index 77e5c161a..dc273efb1 100644 --- a/docs/user/reference/cli/azldev_completion.md +++ b/docs/user/reference/cli/azldev_completion.md @@ -30,6 +30,7 @@ See each sub-command's help for details on how to use the generated script. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_bash.md b/docs/user/reference/cli/azldev_completion_bash.md index 823c98d57..5e9ddcaea 100644 --- a/docs/user/reference/cli/azldev_completion_bash.md +++ b/docs/user/reference/cli/azldev_completion_bash.md @@ -53,6 +53,7 @@ azldev completion bash -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_fish.md b/docs/user/reference/cli/azldev_completion_fish.md index d0b94d9a8..537f2481f 100644 --- a/docs/user/reference/cli/azldev_completion_fish.md +++ b/docs/user/reference/cli/azldev_completion_fish.md @@ -44,6 +44,7 @@ azldev completion fish [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_powershell.md b/docs/user/reference/cli/azldev_completion_powershell.md index dea18386e..7c66365c1 100644 --- a/docs/user/reference/cli/azldev_completion_powershell.md +++ b/docs/user/reference/cli/azldev_completion_powershell.md @@ -41,6 +41,7 @@ azldev completion powershell [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_zsh.md b/docs/user/reference/cli/azldev_completion_zsh.md index d41ec7610..8f65002a1 100644 --- a/docs/user/reference/cli/azldev_completion_zsh.md +++ b/docs/user/reference/cli/azldev_completion_zsh.md @@ -55,6 +55,7 @@ azldev completion zsh [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component.md b/docs/user/reference/cli/azldev_component.md index 525c63a71..6e73f14f3 100644 --- a/docs/user/reference/cli/azldev_component.md +++ b/docs/user/reference/cli/azldev_component.md @@ -33,6 +33,7 @@ components defined in the project configuration. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_add.md b/docs/user/reference/cli/azldev_component_add.md index bce63e5ed..e01bd9003 100644 --- a/docs/user/reference/cli/azldev_component_add.md +++ b/docs/user/reference/cli/azldev_component_add.md @@ -46,6 +46,7 @@ azldev component add [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_build.md b/docs/user/reference/cli/azldev_component_build.md index 2086f683a..f25436c8b 100644 --- a/docs/user/reference/cli/azldev_component_build.md +++ b/docs/user/reference/cli/azldev_component_build.md @@ -77,6 +77,7 @@ azldev component build [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_changed.md b/docs/user/reference/cli/azldev_component_changed.md index ce88bd3fe..d8908e341 100644 --- a/docs/user/reference/cli/azldev_component_changed.md +++ b/docs/user/reference/cli/azldev_component_changed.md @@ -73,6 +73,7 @@ azldev component changed [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_diff-sources.md b/docs/user/reference/cli/azldev_component_diff-sources.md index afd471ac7..33930191b 100644 --- a/docs/user/reference/cli/azldev_component_diff-sources.md +++ b/docs/user/reference/cli/azldev_component_diff-sources.md @@ -40,6 +40,7 @@ azldev component diff-sources [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_history.md b/docs/user/reference/cli/azldev_component_history.md index 00c58cd58..c5d4d90f7 100644 --- a/docs/user/reference/cli/azldev_component_history.md +++ b/docs/user/reference/cli/azldev_component_history.md @@ -66,6 +66,7 @@ azldev component history [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_list.md b/docs/user/reference/cli/azldev_component_list.md index 2900def00..0335f4fd8 100644 --- a/docs/user/reference/cli/azldev_component_list.md +++ b/docs/user/reference/cli/azldev_component_list.md @@ -55,6 +55,7 @@ azldev component list [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 590286e75..4d4596d92 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -60,6 +60,7 @@ azldev component prepare-sources [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_query.md b/docs/user/reference/cli/azldev_component_query.md index 688fdc35e..3b5214718 100644 --- a/docs/user/reference/cli/azldev_component_query.md +++ b/docs/user/reference/cli/azldev_component_query.md @@ -40,7 +40,6 @@ azldev component query [flags] -p, --component stringArray Component name pattern -g, --component-group stringArray Component group name -h, --help help for query - --skip-lock-validation skip lock file consistency checks -s, --spec-path stringArray Spec path ``` @@ -58,6 +57,7 @@ azldev component query [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 1ded12dec..6ffcbc4d9 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -80,6 +80,7 @@ azldev component render [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_update.md b/docs/user/reference/cli/azldev_component_update.md index 911445799..574556b5d 100644 --- a/docs/user/reference/cli/azldev_component_update.md +++ b/docs/user/reference/cli/azldev_component_update.md @@ -78,6 +78,7 @@ azldev component update [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config.md b/docs/user/reference/cli/azldev_config.md index f162e679d..5a1310dfe 100644 --- a/docs/user/reference/cli/azldev_config.md +++ b/docs/user/reference/cli/azldev_config.md @@ -31,6 +31,7 @@ JSON schema used for validating TOML config files. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config_dump.md b/docs/user/reference/cli/azldev_config_dump.md index 8642c8a5c..015c61f94 100644 --- a/docs/user/reference/cli/azldev_config_dump.md +++ b/docs/user/reference/cli/azldev_config_dump.md @@ -51,6 +51,7 @@ azldev config dump [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config_generate-schema.md b/docs/user/reference/cli/azldev_config_generate-schema.md index d7f70cc02..9ff83f63a 100644 --- a/docs/user/reference/cli/azldev_config_generate-schema.md +++ b/docs/user/reference/cli/azldev_config_generate-schema.md @@ -45,6 +45,7 @@ azldev config generate-schema [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs.md b/docs/user/reference/cli/azldev_docs.md index 13b148044..d1409553e 100644 --- a/docs/user/reference/cli/azldev_docs.md +++ b/docs/user/reference/cli/azldev_docs.md @@ -31,6 +31,7 @@ command tree, suitable for inclusion in the user guide. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent.md b/docs/user/reference/cli/azldev_docs_agent.md index 20d9c28cc..eb45bcf16 100644 --- a/docs/user/reference/cli/azldev_docs_agent.md +++ b/docs/user/reference/cli/azldev_docs_agent.md @@ -33,6 +33,7 @@ reference so that agents always load the guidance that ships with the binary. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent_install.md b/docs/user/reference/cli/azldev_docs_agent_install.md index 794b9f496..18eadd0eb 100644 --- a/docs/user/reference/cli/azldev_docs_agent_install.md +++ b/docs/user/reference/cli/azldev_docs_agent_install.md @@ -24,11 +24,13 @@ read-only 'docs-agent-show' MCP tool for the full, always-current skill. Pass --full to inline the complete skill instead, for environments without the azldev MCP server. -Directory paths in the emitted content (such as the lock and rendered-spec -directories) are resolved from the loaded azldev.toml, falling back to azldev's -built-in defaults when no configuration is found. The bindings reflect the project -azldev runs in, so pair --output-dir with -C pointing at the target repository when -scaffolding a different repo. +Directory paths in the emitted content (such as the lock, generated +upstream-commit, and rendered-spec directories) are resolved from the loaded +azldev.toml, falling back to azldev's built-in defaults when no configuration is +found. The emitted content also reflects the mode azldev runs in, so pass +--without-lockfile to describe the lock-file-free workflow. The bindings reflect +the project azldev runs in, so pair --output-dir with -C pointing at the target +repository when scaffolding a different repo. ``` azldev docs agent install [flags] @@ -67,6 +69,7 @@ azldev docs agent install [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent_show.md b/docs/user/reference/cli/azldev_docs_agent_show.md index be58c650b..4d03a7f0f 100644 --- a/docs/user/reference/cli/azldev_docs_agent_show.md +++ b/docs/user/reference/cli/azldev_docs_agent_show.md @@ -50,6 +50,7 @@ azldev docs agent show [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_markdown.md b/docs/user/reference/cli/azldev_docs_markdown.md index 15a9631ec..de9864f18 100644 --- a/docs/user/reference/cli/azldev_docs_markdown.md +++ b/docs/user/reference/cli/azldev_docs_markdown.md @@ -52,6 +52,7 @@ azldev docs markdown [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image.md b/docs/user/reference/cli/azldev_image.md index 92f123151..2235a6268 100644 --- a/docs/user/reference/cli/azldev_image.md +++ b/docs/user/reference/cli/azldev_image.md @@ -32,6 +32,7 @@ can be customized using Azure Linux Image Customizer. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_boot.md b/docs/user/reference/cli/azldev_image_boot.md index cf88b6041..b6813a392 100644 --- a/docs/user/reference/cli/azldev_image_boot.md +++ b/docs/user/reference/cli/azldev_image_boot.md @@ -96,6 +96,7 @@ azldev image boot [IMAGE_NAME] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_build.md b/docs/user/reference/cli/azldev_image_build.md index 2f0db4daf..ab8d903af 100644 --- a/docs/user/reference/cli/azldev_image_build.md +++ b/docs/user/reference/cli/azldev_image_build.md @@ -54,6 +54,7 @@ azldev image build [image-name] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_customize.md b/docs/user/reference/cli/azldev_image_customize.md index 6f09cfc95..dfeaf4f5e 100644 --- a/docs/user/reference/cli/azldev_image_customize.md +++ b/docs/user/reference/cli/azldev_image_customize.md @@ -55,6 +55,7 @@ azldev image customize [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_inject-files.md b/docs/user/reference/cli/azldev_image_inject-files.md index 154420764..32d3795c5 100644 --- a/docs/user/reference/cli/azldev_image_inject-files.md +++ b/docs/user/reference/cli/azldev_image_inject-files.md @@ -46,6 +46,7 @@ azldev image inject-files [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_list.md b/docs/user/reference/cli/azldev_image_list.md index 4b8cdfb88..a96aee0e6 100644 --- a/docs/user/reference/cli/azldev_image_list.md +++ b/docs/user/reference/cli/azldev_image_list.md @@ -48,6 +48,7 @@ azldev image list [image-name-pattern...] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index 2816ac3c5..e6f965a9d 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -85,6 +85,7 @@ azldev image test IMAGE_NAME [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_package.md b/docs/user/reference/cli/azldev_package.md index d3fefb25f..103db352d 100644 --- a/docs/user/reference/cli/azldev_package.md +++ b/docs/user/reference/cli/azldev_package.md @@ -32,6 +32,7 @@ publish channel assignments derived from package groups and component overrides. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_package_list.md b/docs/user/reference/cli/azldev_package_list.md index 7fd3bd0e4..6b9754eab 100644 --- a/docs/user/reference/cli/azldev_package_list.md +++ b/docs/user/reference/cli/azldev_package_list.md @@ -81,6 +81,7 @@ azldev package list [package-name...] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project.md b/docs/user/reference/cli/azldev_project.md index fed57377e..e2f2c332e 100644 --- a/docs/user/reference/cli/azldev_project.md +++ b/docs/user/reference/cli/azldev_project.md @@ -31,6 +31,7 @@ as an Azure Linux project with a basic configuration. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project_init.md b/docs/user/reference/cli/azldev_project_init.md index 5d25f7ee3..d133018c4 100644 --- a/docs/user/reference/cli/azldev_project_init.md +++ b/docs/user/reference/cli/azldev_project_init.md @@ -42,6 +42,7 @@ azldev project init [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project_new.md b/docs/user/reference/cli/azldev_project_new.md index 3baa4098b..1a615d5f7 100644 --- a/docs/user/reference/cli/azldev_project_new.md +++ b/docs/user/reference/cli/azldev_project_new.md @@ -45,6 +45,7 @@ azldev project new PATH [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo.md b/docs/user/reference/cli/azldev_repo.md index e9132d0b8..384e4b029 100644 --- a/docs/user/reference/cli/azldev_repo.md +++ b/docs/user/reference/cli/azldev_repo.md @@ -32,6 +32,7 @@ under one or more URL prefixes. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo_query.md b/docs/user/reference/cli/azldev_repo_query.md index f45a51ce5..b00fb5f74 100644 --- a/docs/user/reference/cli/azldev_repo_query.md +++ b/docs/user/reference/cli/azldev_repo_query.md @@ -73,6 +73,7 @@ azldev repo query [flags] -- -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_version.md b/docs/user/reference/cli/azldev_version.md index f24b9ea1c..9d8707559 100644 --- a/docs/user/reference/cli/azldev_version.md +++ b/docs/user/reference/cli/azldev_version.md @@ -36,6 +36,7 @@ azldev version -O json -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index 19abd9722..8aff496cd 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -38,6 +38,8 @@ Glob patterns that match no files are silently ignored. Literal filenames (no wi Includes are resolved recursively — included files can themselves declare further includes. For a detailed explanation of load order and merge semantics, see [Configuration System](../../explanation/config-system.md). +> **Preview:** With the global `--without-lockfile` flag, the generated upstream-commit files must be included before any component-specific TOML configuration, so that later component definitions supply the remaining `spec` fields and may explicitly override a generated pin. See [Preview the Lock-File-Free Mode](../../how-to/preview-without-lockfile.md). + ## Minimal Example A minimal root config file that includes distro definitions and a project: diff --git a/internal/app/azldev/agentskill/agentskill.go b/internal/app/azldev/agentskill/agentskill.go index 65692769a..eef80fa28 100644 --- a/internal/app/azldev/agentskill/agentskill.go +++ b/internal/app/azldev/agentskill/agentskill.go @@ -9,6 +9,7 @@ import ( "fmt" "path" "slices" + "strings" "text/template" ) @@ -31,16 +32,27 @@ const ( ) // The embedded templates rendered into the emitted files and the served skill -// documents. +// documents. Templates under 'content/withoutlockfile' replace the same-named +// template when lock-file-free mode is selected. // -//go:embed content/*.tmpl +//go:embed content/*.tmpl content/withoutlockfile/*.tmpl var content embed.FS -// templates holds all parsed templates, keyed by their base file name. +// templates holds all parsed templates for azldev's default mode, keyed by their +// base file name. // //nolint:gochecknoglobals // parsed templates are effectively constant and safe for concurrent use. var templates = template.Must(template.ParseFS(content, "content/*.tmpl")) +// withoutLockfileTemplates holds the same templates with the lock-file-free +// variants layered on top: parsing a template whose base name already exists +// replaces it, so only the differing documents need their own file. +// +//nolint:gochecknoglobals // parsed templates are effectively constant and safe for concurrent use. +var withoutLockfileTemplates = template.Must(template.ParseFS( + content, "content/*.tmpl", "content/withoutlockfile/*.tmpl", +)) + // Skill describes a single emitted Agent Skill. type Skill struct { // Name is the stable base identifier (lowercase, hyphen-delimited). It is the @@ -156,14 +168,81 @@ var skills = []Skill{ }, } -// Skills returns the registered skills in emission order. -func Skills() []Skill { - return slices.Clone(skills) +// withoutLockfileSkills replaces registry entries when lock-file-free mode is +// selected, keyed by the name of the default-mode skill it replaces. Everything +// else — order, remaining skills, and the emitted layout — is shared. +// +//nolint:gochecknoglobals // effectively a constant registry of the mode's skills. +var withoutLockfileSkills = map[string]Skill{ + "azldev": { + Name: "azldev", + Description: "Read this before running azldev or editing azldev config, and whenever working " + + "in a repo that contains an azldev.toml file; do not guess azldev's commands or config. " + + "Explains how to use the azldev CLI to build a distro from TOML config, including the core " + + "concepts (components, overlays, distros, rendered specs, upstream commit config), running " + + "azldev (repo root or -C, plus the -q and -O json flags), the common commands, and where to " + + "go for each workflow. Triggers include azldev, comp build, comp render, " + + "comp refresh-upstream-commit, build a component, add a component, distro config.", + bodyTemplate: "azldev.md.tmpl", + }, + "azldev-update-component": { + Name: "azldev-refresh-upstream-commit", + Description: "Read this before finalizing a component change, changing source resolution, or " + + "editing generated upstream commit TOML by hand. Explains how to refresh commits with " + + "'azldev comp refresh-upstream-commit', covering when to refresh versus render, the " + + "update/render/commit/re-render/amend workflow, and per-component versus -a refresh. " + + "Triggers include comp refresh-upstream-commit, refresh upstream commit, bump pin, change " + + "snapshot, upstream distro, commit drift, version bump, finalize component.", + bodyTemplate: "refresh-upstream-commit.md.tmpl", + }, } -// FindSkill returns the registered skill with the given name. -func FindSkill(name string) (Skill, error) { - for _, skill := range skills { +// updateComponentSkillName is the default mode's finalization skill; lock-file-free +// mode replaces it with the refresh-upstream-commit skill. +const updateComponentSkillName = "azldev-update-component" + +// Catalog exposes the skills and instruction files for one of azldev's modes. The +// registries are shared; only the documents and pointers that describe how resolved +// component state is maintained differ. +type Catalog struct { + withoutLockfile bool +} + +// NewCatalog returns the catalog for the selected mode. Pass true to describe +// lock-file-free mode, as selected by the global '--without-lockfile' flag. +func NewCatalog(withoutLockfile bool) Catalog { + return Catalog{withoutLockfile: withoutLockfile} +} + +// templates returns the parsed template set for the catalog's mode. +func (c Catalog) templates() *template.Template { + if c.withoutLockfile { + return withoutLockfileTemplates + } + + return templates +} + +// Skills returns the catalog's skills in emission order. +func (c Catalog) Skills() []Skill { + result := slices.Clone(skills) + + if !c.withoutLockfile { + return result + } + + for idx := range result { + if replacement, ok := withoutLockfileSkills[result[idx].Name]; ok { + result[idx] = replacement + } + } + + return result +} + +// FindSkill returns the catalog's skill with the given name. +func (c Catalog) FindSkill(name string) (Skill, error) { + for _, skill := range c.Skills() { if skill.Name == name { return skill, nil } @@ -172,6 +251,48 @@ func FindSkill(name string) (Skill, error) { return Skill{}, fmt.Errorf("unknown skill %#q", name) } +// Instructions returns the catalog's instruction files in emission order. +func (c Catalog) Instructions() []Instruction { + result := slices.Clone(instructions) + for idx := range result { + result[idx].Skills = slices.Clone(result[idx].Skills) + } + + if !c.withoutLockfile { + return result + } + + refresh := withoutLockfileSkills[updateComponentSkillName] + + for instIdx := range result { + if replacement, ok := withoutLockfileInstructions[result[instIdx].Name]; ok { + result[instIdx].Description = replacement.Description + } + + for skillIdx := range result[instIdx].Skills { + pointer := &result[instIdx].Skills[skillIdx] + if pointer.Skill != updateComponentSkillName { + continue + } + + pointer.Skill = refresh.Name + pointer.Purpose = strings.ReplaceAll(pointer.Purpose, "lock", "upstream commit") + } + } + + return result +} + +// Skills returns the registered skills in emission order for azldev's default mode. +func Skills() []Skill { + return NewCatalog(false).Skills() +} + +// FindSkill returns the registered skill with the given name in azldev's default mode. +func FindSkill(name string) (Skill, error) { + return NewCatalog(false).FindSkill(name) +} + // SkillPointer names a skill an instruction file points at, together with a short // purpose describing when to read it ("read the `azldev-overlays` skill to add or change // overlays"). @@ -268,14 +389,24 @@ var instructions = []Instruction{ }, } -// Instructions returns the registered instruction files in emission order. -func Instructions() []Instruction { - result := slices.Clone(instructions) - for i := range result { - result[i].Skills = slices.Clone(result[i].Skills) - } +// withoutLockfileInstructions replaces instruction descriptions when lock-file-free +// mode is selected, keyed by instruction name. Skill pointers are rewritten +// automatically, so only the free-form trigger text lives here. +// +//nolint:gochecknoglobals // effectively a constant registry of the mode's instructions. +var withoutLockfileInstructions = map[string]Instruction{ + SkillName: { + Description: "This repo is an azldev distro project (azldev.toml present). Before running azldev " + + "or editing its config, load the azldev skill; do not guess azldev's commands or config. " + + "Triggers include azldev, comp build, comp render, comp refresh-upstream-commit, build a " + + "component, add a component, distro config.", + }, +} - return result +// Instructions returns the registered instruction files in emission order for +// azldev's default mode. +func Instructions() []Instruction { + return NewCatalog(false).Instructions() } // Layout controls where emitted skill files are written in a target repository. @@ -322,8 +453,13 @@ type Command struct { // accurate for a default project even with no configuration present. type Bindings struct { // LockDir is the repo-relative directory holding per-component lock files. + // Only meaningful in azldev's default mode. LockDir string + // UpstreamCommitsDir is the repo-relative directory holding the generated + // per-component commit TOMLs. Only meaningful in lock-file-free mode. + UpstreamCommitsDir string + // RenderedSpecsDir is the repo-relative directory holding rendered component specs. RenderedSpecsDir string @@ -356,7 +492,7 @@ type EmittedFile struct { Content string `json:"-"` } -func renderSkill(templateName string, skill Skill, params Params) (string, error) { +func (c Catalog) renderSkill(templateName string, skill Skill, params Params) (string, error) { var buf bytes.Buffer data := struct { @@ -369,7 +505,7 @@ func renderSkill(templateName string, skill Skill, params Params) (string, error ShowSkillToolName: ShowSkillToolName, } - err := templates.ExecuteTemplate(&buf, templateName, data) + err := c.templates().ExecuteTemplate(&buf, templateName, data) if err != nil { return "", fmt.Errorf("failed to render agent skill template %#q:\n%w", templateName, err) } @@ -377,8 +513,8 @@ func renderSkill(templateName string, skill Skill, params Params) (string, error return buf.String(), nil } -func renderInstruction(inst Instruction, params Params) (string, error) { - if err := validateInstruction(inst); err != nil { +func (c Catalog) renderInstruction(inst Instruction, params Params) (string, error) { + if err := c.validateInstruction(inst); err != nil { return "", err } @@ -398,7 +534,7 @@ func renderInstruction(inst Instruction, params Params) (string, error) { Instruction: inst, } - err = templates.ExecuteTemplate(&buf, "instruction-wrapper.md.tmpl", data) + err = c.templates().ExecuteTemplate(&buf, "instruction-wrapper.md.tmpl", data) if err != nil { return "", fmt.Errorf("failed to render instruction template for %#q:\n%w", inst.Name, err) } @@ -406,13 +542,13 @@ func renderInstruction(inst Instruction, params Params) (string, error) { return buf.String(), nil } -func validateInstruction(inst Instruction) error { +func (c Catalog) validateInstruction(inst Instruction) error { if len(inst.Skills) == 0 { return fmt.Errorf("instruction %#q must reference at least one skill", inst.Name) } for _, pointer := range inst.Skills { - if _, err := FindSkill(pointer.Skill); err != nil { + if _, err := c.FindSkill(pointer.Skill); err != nil { return fmt.Errorf("instruction %#q references unknown skill %#q:\n%w", inst.Name, pointer.Skill, err) } @@ -442,13 +578,18 @@ func renderInline(name, text string, params Params) (string, error) { // SkillDocument renders the full document for the named skill. It is served // verbatim by the read-only MCP tool and by 'azldev docs agent show'. The default // layout is used since a served document has no on-disk directory. -func SkillDocument(name string, params Params) (string, error) { - skill, err := FindSkill(name) +func (c Catalog) SkillDocument(name string, params Params) (string, error) { + skill, err := c.FindSkill(name) if err != nil { return "", err } - return renderSkill(skill.bodyTemplate, skill, params) + return c.renderSkill(skill.bodyTemplate, skill, params) +} + +// SkillDocument renders the named skill's document for azldev's default mode. +func SkillDocument(name string, params Params) (string, error) { + return NewCatalog(false).SkillDocument(name, params) } // Files renders the set of agent files to write into a target repository using the @@ -456,16 +597,18 @@ func SkillDocument(name string, params Params) (string, error) { // skill document instead of a light MCP wrapper (useful when the azldev MCP server // is not available in the target environment). Instruction files are always light // wrappers that point at the relevant skills. -func Files(layout Layout, params Params, full bool) ([]EmittedFile, error) { - files := make([]EmittedFile, 0, len(skills)+len(instructions)) +func (c Catalog) Files(layout Layout, params Params, full bool) ([]EmittedFile, error) { + catalogSkills := c.Skills() + catalogInstructions := c.Instructions() + files := make([]EmittedFile, 0, len(catalogSkills)+len(catalogInstructions)) - for _, skill := range skills { + for _, skill := range catalogSkills { templateName := "skill-wrapper.md.tmpl" if full { templateName = skill.bodyTemplate } - rendered, err := renderSkill(templateName, skill, params) + rendered, err := c.renderSkill(templateName, skill, params) if err != nil { return nil, err } @@ -473,8 +616,8 @@ func Files(layout Layout, params Params, full bool) ([]EmittedFile, error) { files = append(files, EmittedFile{RelPath: layout.SkillFile(skill), Content: rendered}) } - for _, inst := range instructions { - rendered, err := renderInstruction(inst, params) + for _, inst := range catalogInstructions { + rendered, err := c.renderInstruction(inst, params) if err != nil { return nil, err } @@ -484,3 +627,8 @@ func Files(layout Layout, params Params, full bool) ([]EmittedFile, error) { return files, nil } + +// Files renders the agent files for azldev's default mode. +func Files(layout Layout, params Params, full bool) ([]EmittedFile, error) { + return NewCatalog(false).Files(layout, params, full) +} diff --git a/internal/app/azldev/agentskill/agentskill_internal_test.go b/internal/app/azldev/agentskill/agentskill_internal_test.go index 1231690c6..dbcaba323 100644 --- a/internal/app/azldev/agentskill/agentskill_internal_test.go +++ b/internal/app/azldev/agentskill/agentskill_internal_test.go @@ -31,7 +31,7 @@ func TestRenderInstructionRejectsInvalidSkillPointers(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := renderInstruction(Instruction{ + _, err := NewCatalog(false).renderInstruction(Instruction{ Name: "test-instruction", Skills: test.skills, }, Params{}) diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 123aca4f0..8a65a7b7c 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -416,3 +416,64 @@ func TestOverlayMetadataSkillCoversSchemaEnums(t *testing.T) { } } } + +// TestCatalog_SkillsByMode verifies that each mode advertises the skill describing +// how it maintains resolved component state, and only that one. +func TestCatalog_SkillsByMode(t *testing.T) { + defaultNames := skillNamesOf(agentskill.NewCatalog(false)) + assert.Contains(t, defaultNames, "azldev-update-component") + assert.NotContains(t, defaultNames, "azldev-refresh-upstream-commit") + + withoutLockfileNames := skillNamesOf(agentskill.NewCatalog(true)) + assert.Contains(t, withoutLockfileNames, "azldev-refresh-upstream-commit") + assert.NotContains(t, withoutLockfileNames, "azldev-update-component") + + // The two catalogs otherwise describe the same set of skills, in the same order. + assert.Len(t, withoutLockfileNames, len(defaultNames)) +} + +// TestCatalog_SkillContentByMode verifies that the rendered skill bodies describe +// the mode's own workflow. +func TestCatalog_SkillContentByMode(t *testing.T) { + params := testParams() + params.UpstreamCommitsDir = "base/upstream-commits" + + defaultDoc, err := agentskill.NewCatalog(false).SkillDocument("azldev-add-component", params) + require.NoError(t, err) + assert.Contains(t, defaultDoc, "azldev comp update") + assert.Contains(t, defaultDoc, "locks/.lock") + + withoutLockfileDoc, err := agentskill.NewCatalog(true).SkillDocument("azldev-add-component", params) + require.NoError(t, err) + assert.Contains(t, withoutLockfileDoc, "azldev comp refresh-upstream-commit") + assert.Contains(t, withoutLockfileDoc, "base/upstream-commits/.toml") + assert.NotContains(t, withoutLockfileDoc, "azldev comp update") +} + +// TestCatalog_InstructionsPointAtModeSkills verifies that instruction wrappers point +// at the skill that exists in the active mode. +func TestCatalog_InstructionsPointAtModeSkills(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + catalog := agentskill.NewCatalog(withoutLockfile) + names := skillNamesOf(catalog) + + for _, inst := range catalog.Instructions() { + for _, pointer := range inst.Skills { + assert.Contains(t, names, pointer.Skill, + "instruction %q points at a skill that is not registered in this mode", inst.Name) + } + } + } +} + +// skillNamesOf returns the names of the catalog's skills. +func skillNamesOf(catalog agentskill.Catalog) []string { + skills := catalog.Skills() + names := make([]string, 0, len(skills)) + + for _, skill := range skills { + names = append(names, skill.Name) + } + + return names +} diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/add-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/add-component.md.tmpl new file mode 100644 index 000000000..27c94feb6 --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/add-component.md.tmpl @@ -0,0 +1,93 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Add a component + +## Before you start + +Confirm the component does not already exist: + +```sh +azldev comp list -p -q -O json +``` + +### Inspect the upstream spec first + +The reliable way to see what you are importing (direct web fetches of upstream +dist-git often fail bot detection): + +1. Add a bare entry so azldev can resolve the component: run `azldev comp add ` + (it appends `[components.]` to the root config file), or hand-add + `[components.]` to an included config file if you want it to live elsewhere. + A bare root entry is *perfect* for initial testing, but real distros will usually + segment the configuration into included files. Once the initial pass is done, + ensure the component is in the right place and remove the root entry. + +2. Generate the initial upstream commit config so source resolution is pinned before inspection: + + ```sh + azldev comp refresh-upstream-commit -p + ``` + +3. Pull the sources without overlays into a scratch dir under the work dir: + + ```sh + azldev comp prep-sources -p --skip-overlays --force -o {{ .WorkDir }}/scratch/ -q + ``` + +4. Read the spec and plan any overlays. + +## Inline vs dedicated file + +- **Inline** — a bare upstream import with no changes stays in a shared config file: + + ```toml + [components.jq] + ``` + +- **Dedicated** — anything that needs overlays, build config, or a local spec gets its + own `/.comp.toml`. Rule of thumb: more than `[components.]` earns a + dedicated file. An `includes = ["**/*.comp.toml"]` glob picks it up automatically. + +`azldev comp add [...]` adds bare `[components.]` entries that inherit +the distro defaults; it writes to the **root** config file and does not scaffold spec or +source files. If your distro keeps components in included or dedicated files, move the +entry there afterward. + +## Customize + +For spec source types and overlays, read the `azldev-comp-toml` and `azldev-overlays` skills. Key +points when adding a component: + +- Prefer **overlays** over forking the spec — overlays get upstream updates for free. + Forking a spec is a last resort and a long-term maintenance commitment. Get explicit + user sign-off first, document every change, and keep the delta minimal. +- Every overlay needs a `description` explaining why it is needed. +- Keep `%check` enabled. Disable it only as a last resort via `build.check.skip = true` + with a required `build.check.skip_reason` (see the `azldev-build-component` skill). + +## Validate + +The generated config pins the upstream revision you inspected. Refresh it after the +component inputs settle, then validate: + +```sh +azldev comp refresh-upstream-commit -p # resolve and write generated upstream commit config +azldev comp render -p # apply overlays and write the rendered spec +azldev comp diff-sources -p # see exactly what the overlays change +azldev comp build -p # build the RPMs +``` + +Inspect the rendered spec under `{{ .RenderedSpecsDir }}/`. A new component always needs a +smoke test — see the `azldev-build-component` and `azldev-mock` skills. + +After all component inputs are final, run `azldev comp refresh-upstream-commit -p ` again and +re-render. Stage the component definition and sources, `{{ .UpstreamCommitsDir }}/.toml`, and +the rendered output before committing. Then re-render, stage +`{{ .RenderedSpecsDir }}///`, and amend the commit so `%changelog` and +`Release:` reflect it. See the `azldev-refresh-upstream-commit` skill for the complete +finalization workflow. + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl new file mode 100644 index 000000000..c1f87f153 --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -0,0 +1,78 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Using azldev + +azldev builds a Linux distribution from TOML configuration. It imports RPM specs +from an upstream distro and customizes them with an overlay system — no spec +forking. Components render to specs and sidecar build inputs, then build into RPMs; +images assemble RPMs into bootable artifacts. + +## Orient yourself + +- Run azldev from the repo root (where `azldev.toml` lives), or pass `-C `. +- Global, agent-friendly flags: **`-q`** (quiet) and **`-O json`** (machine-readable + output). They work on every command. +- Config is a stitched TOML hierarchy: `azldev.toml` includes the distro and project + config, which include the component files (`**/*.comp.toml`) into one namespace. + +## Core concepts + +- **Component** — a unit of packaging that renders to a spec and sidecar build inputs, + then builds into one or more RPMs. Its spec source is upstream (default), a pinned + upstream distro/version, or a local spec. +- **Overlay** — a semantic patch applied to a spec or source file at render time, so + you customize upstream without forking it. +- **Distro** — a named build target (`*.distro.toml`) with upstream URIs, release + versions, and build inputs. +- **Rendered spec** — the generated `.spec` after overlays; a build input, never + hand-edited. +- **Upstream commit config** — generated component TOML pinning the resolved upstream commit. + +## Common commands + +Top-level commands in this build: + +{{ range .TopLevelCommands }}- `azldev {{ .Name }}` — {{ .Short }} +{{ end }} +Everyday tasks (add `-q -O json` when scripting): + +| Task | Command | +| --- | --- | +| List components | `azldev comp list -a` | +| Inspect one component | `azldev comp list -p ` | +| Add a component | `azldev comp add` | +| Build a component | `azldev comp build -p ` | +| Render specs | `azldev comp render -p ` (or `-a`) | +| Refresh an upstream commit | `azldev comp refresh-upstream-commit -p ` | +| List / build images | `azldev image list` / `azldev image build` | + +`comp` is an alias for `component`. Always confirm current syntax with +`azldev --help`. + +The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and +mock helpers; it is intentionally omitted from normal help. + +## Where to go next + +- Add a new component — read the `azldev-add-component` skill. +- Edit or review a component's TOML — read the `azldev-comp-toml` skill. +- Add or change overlays — read the `azldev-overlays` skill. +- Annotate an overlay's intent (category, upstream status, provenance) — read the `azldev-overlay-metadata` skill. +- Build, iterate, and debug a component — read the `azldev-build-component` skill. +- Build, boot, and register images — read the `azldev-image` skill. +- Refresh an upstream commit and finalize for a PR — read the `azldev-refresh-upstream-commit` skill. +- Remove a component — read the `azldev-remove-component` skill. +- Test built RPMs in a chroot — read the `azldev-mock` skill. + +## Golden rules + +- **Never edit generated output** — rendered specs and the output/work/log dirs are + produced by azldev. Change the source config and re-render. +- **Refresh upstream commits after source-resolution changes** — run `azldev comp refresh-upstream-commit` after + changing a commit pin, upstream distro/version, or snapshot. +- **Every overlay needs a `description`** explaining why the change is needed. + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl new file mode 100644 index 000000000..cac327d99 --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -0,0 +1,60 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Build and debug a component + +**Never install built RPMs on your host** — they target the distro, not your dev +machine. Test them in a chroot with the `azldev-mock` skill. Building and testing are separate +steps: `azldev comp build` produces RPMs; it does not test them. + +## Build + +```sh +azldev comp build -p # one component +azldev comp build -p -p --local-repo-with-publish # chain deps via a local repo +azldev comp build -p --local-repo # rebuild against a populated repo +``` + +Build foundational packages before their dependents. RPMs land in the project's +configured output directory (`out` by default). `-q` quiets output but hides build +progress — use it only for inner-loop builds you expect to succeed. + +## The inner loop + +investigate → modify → render → build → test → inspect + +| Step | Command | +| --- | --- | +| Investigate | read the rendered spec under `{{ .RenderedSpecsDir }}/`, or `azldev comp diff-sources -p ` | +| Modify | edit the component's `.comp.toml` (see the `azldev-overlays` and `azldev-comp-toml` skills) | +| Verify | `azldev comp render -p ` (fast — skips source tarballs) | +| Build | `azldev comp build -p ` | +| Test | `azldev adv mock shell --add-package ` (see the `azldev-mock` skill) | +| Inspect | `azldev comp build -p --preserve-buildenv always`, then a mock shell | + +Prefer `comp render` for quick overlay verification; use `comp diff-sources` to see the +exact overlay effect (it fetches sources once, applies overlays to a copy, and diffs the +two trees). Builds can be slow — set generous timeouts. + +Finalize with `azldev comp refresh-upstream-commit -p ` before opening a PR (see the +`azldev-refresh-upstream-commit` skill). + +## Debugging build failures + +1. **Render error mentioning a non-standard `Release` tag** — a release-calculation + issue; see the `azldev-comp-toml` skill. +2. **Overlay did not apply as expected** — `azldev comp diff-sources -p ` shows + what the overlays actually change. +3. **Inspect the build environment** — `azldev comp build -p --preserve-buildenv + on-failure` (values `on-failure`, `always`, `never`), then enter a mock shell. +4. **Failing `%check`** — fix the tests first (root cause, upstream patches, targeted + fixes). Only as a last resort, disable with `build.check.skip = true` and a required + `build.check.skip_reason` explaining what fails, why it cannot be fixed, and whether it + is temporary. A transient `--no-check` build flag exists for one-off local builds. + +Per-component build tweaks (`build.defines`, `build.without`) live in the `.comp.toml` — +see the `azldev-comp-toml` skill. + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl new file mode 100644 index 000000000..0e33a9e6d --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -0,0 +1,142 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Component definition files (`*.comp.toml`) + +A component definition tells azldev where a package's spec comes from and how to +customize it for your distro. Every component lives under `[components.]`. + +Get the authoritative, always-current field list from the schema: + +```sh +azldev config generate-schema +``` + +## Structure + +A bare entry inherits everything from your distro's defaults — most upstream packages +need nothing more: + +```toml +[components.curl] +``` + +Add sub-tables only for what you change. The fields you will reach for most: + +| Field | Purpose | +| --- | --- | +| `spec` | where the spec comes from (see below) | +| `overlays` / `overlay-files` | targeted spec/source edits (see the `azldev-overlays` skill) | +| `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | +| `release.calculation` | how the `Release` tag is managed | +| `render.skip-file-filter` | rendering edge-case escape hatch | + +## Spec source + +The `spec` field selects where the spec is fetched from. When omitted, the component +inherits the distro default (normally an upstream import). + +```toml +# Upstream import (the usual case) — inherits the distro's upstream version +[components.curl] + +# Upstream, but pinned to a specific upstream distro/version +[components.curl] +spec = { type = "upstream", upstream-distro = { name = "fedora", version = "rawhide" } } + +# Upstream package whose name differs from the component name +[components.mydistro-rpm-config] +spec = { type = "upstream", upstream-name = "redhat-rpm-config" } + +# Local spec that lives in your repo (not imported from an upstream distro) +[components.mydistro-release] +spec = { type = "local", path = "mydistro-release.spec" } +``` + +## Build configuration + +```toml +[components.mypackage.build] +defines = { rhel = "11" } # override RPM macros +with = ["feature_x"] # enable %bcond_with conditionals +without = ["plugin_rhsm"] # disable %bcond_with conditionals +``` + +## Release calculation + +`release.calculation` controls the `Release:` tag. There are four modes: + +- `auto` (default) — auto-detect whether the spec uses `%autorelease` or a static + release and handle it accordingly. Correct for most packages. +- `autorelease` — force `%autorelease` handling (use when auto-detection misreads a + spec that wraps `%autorelease` in a conditional). +- `static` — force static-integer handling and bump the integer on render (the + inverse of `autorelease`). +- `manual` — you own the `Release:` value. Use this only when render fails with a + "non-standard Release tag" error. **A `manual` component is not bumped by the + render/commit/amend cycle, so increment its release yourself in the same change** + (see the `azldev-refresh-upstream-commit` skill). + +```toml +[components.mypackage.release] +calculation = "manual" +``` + +## Render configuration + +`render.skip-file-filter = true` keeps all source and patch files during render. +azldev normally prunes files not referenced by the rendered spec; set this only for +the rare spec whose `Source`/`Patch` filenames use macros the filter cannot expand. + +## File organization + +- **Inline** — put simple, customization-free components directly in a shared config + file (e.g. `[components.jq]`). +- **Dedicated** — give a component its own `/.comp.toml` once it needs + overlays, build config, or a local spec. Rule of thumb: anything more than + `[components.]` earns a dedicated file. +- A parent config picks up dedicated files through an `includes` glob, for example + `includes = ["**/*.comp.toml"]`. + +## Review checklist + +Start with `azldev comp list -p -q -O json`. When the review needs spec +details, inspect the rendered spec under +`{{ .RenderedSpecsDir }}///`. For a change review, focus on +the diff while checking enough surrounding context to ensure it fits the +component and repository conventions. + +- **Organization:** The component follows the repository's inline-versus-dedicated-file + convention; its name matches upstream or sets `spec.upstream-name`; no stale or + orphaned component files remain. +- **Spec source:** The default upstream source is preferred. Pins explain why they are + needed, and local specs are used only when overlays cannot express the change. +- **Overlays:** Every overlay explains why it exists. Prefer structured overlay types + over regex; scope unavoidable `spec-search-replace` expressions by section and, when + applicable, package, and use TOML literal strings. `spec-search-replace` cannot span + lines. Remove overlays that upstream has made unnecessary. See the `azldev-overlays` skill. +- **Build config:** Defines and bcond overrides are necessary and correspond to the + spec. If `build.check.skip = true`, require a specific `build.check.skip_reason` and + verify that fixing the tests is impractical; skipped `%check` is a last resort. +- **Release mode:** `auto` is the default. Force `autorelease` or `static` only when + auto-detection is wrong. Use `manual` only for a non-standard release tag, and verify + the component increments its release itself. +- **Generated state:** The upstream commit config matches the final component inputs, and rendered output + contains the intended changes without unrelated drift. +- **Testing:** Changes that can affect RPM output were built and smoke-tested in a mock + chroot. Organization, comment, or documentation-only metadata edits do not require a + rebuild when the resolved component inputs are unchanged. + +Report findings by severity: errors for correctness or required-policy violations, +warnings for maintainability risks, and info for optional improvements. Prefer small, +actionable fixes over unrelated cleanup. + +## Documenting changes + +Add a TOML comment explaining *why* a non-obvious field is set (a version pin, a +workaround), and link the upstream commit or bug when the change is based on one. For +overlays, use the overlay `metadata` table instead (see the `azldev-overlays` skill). + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/overlay-metadata.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/overlay-metadata.md.tmpl new file mode 100644 index 000000000..405bb75d1 --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/overlay-metadata.md.tmpl @@ -0,0 +1,177 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Overlay metadata — pick a category and annotate + +Every overlay documents *why* it exists (`description`, **required**) and *what class +of change* it is (`metadata`). When a `[metadata]` block is present it **requires** both +a `category` and an `upstream-status`. This skill covers how to pick the right values, +attach provenance, and write the TOML at the moment you author or review an overlay. + +Metadata is **pure documentation** and never changes the rendered spec. For +overlay *types* and the render-and-inspect loop, read the `azldev-overlays` +skill; this skill is only about the `metadata` table. + +## One `[metadata]` block = one logical change + +A single logical change (a CVE backport, a feature disablement, a Fedora cherry-pick) +may need **several overlays** — e.g. remove a sub-package *and* drop the configure flags +that went with it. The per-file overlay format (`overlay-files`) captures exactly this: +one top-level `[metadata]` table and one or more `[[overlays]]` entries it applies to. +Per-overlay `metadata` inside an overlay file is **rejected** — the file-level block is +the single source of truth. + +So: if you find yourself stamping the *same* metadata on several inline overlays, that is +a signal they are one logical change — move them into a single overlay file with one +`[metadata]` block. + +## When to add metadata + +- **New overlay** (any inline `[[components..overlays]]` block or `[[overlays]]` + entry in an overlay file): add `metadata` with both a `category` and an `upstream-status`. +- **An overlay's intent changed** (e.g. a prune becomes a backport): update the `category` + and `upstream-status` to match the new intent. + +## Step 1 — Pick the category + +Choose exactly one `category` from this closed set (authoritative — matches +`metadata.category` in the schema): + +| `category` | Use when the overlay… | Extra required/expected fields | +|------------|-----------------------|--------------------------------| +| `upstream-backport` | Backports a fix from an upstream source (Fedora dist-git or the component's OSS project) that AZL will inherit once it bumps past the fix. Self-resolves on version bump. | `commits` (≥1 upstream commit URL) — **required**. `upstream-status` must be `upstreamed` or `upstreamable`. | +| `azl-pruning` | Removes content for AZL: unshipped deps, unneeded features, sub-packages, or files. | — | +| `azl-compatibility` | Adapts a component to *how Azure Linux is built and shipped* — build tooling, buildroot, infrastructure, runtime ecosystem — when upstream builds/behaves incorrectly for AZL-specific reasons that are **not** branding, a missing dependency, architecture, or tests (e.g. `azldev` downloader quirks, `rpmdiff` reproducibility, buildroot gaps, Fedora version-skew). | — | +| `azl-temp-workaround` | Temporary workaround explicitly intended to be dropped once an upstream or environmental fix lands. Covers a dependency not yet imported into AZL (or unavailable on a target) **and** any other transient workaround waiting on an external change. | — | +| `azl-branding-policy` | Fedora→Azure Linux identity differences: intentional name/path/vendor conventions **and** spec fixes for upstream code that hard-codes Fedora identity strings (e.g. `_vendor=redhat`, `-redhat-linux[-gnu]` triples, `redhat-linux-build` dirs). Also covers repointing a `Source`/`URL` tag from a Fedora mirror to an Azure Linux one (e.g. `azurelinux-rpm-config`, `golang`). | — | +| `azl-disable-flaky-tests` | Skips tests that fail intermittently / due to environmental flakiness, not a real component bug. | — | +| `azl-disable-unsupported-tests` | Skips tests that cannot meaningfully run in AZL's build/runtime env (need network, root, or unavailable hardware in mock). | — | +| `azl-security-compliance` | Makes FIPS or crypto-policy changes. | — | +| `azl-release-management` | Adjusts release-tag / changelog mechanics. | — | +| `azl-platform-adaptation` | Makes architecture-specific adjustments. | — | + +### Disambiguation tips + +- **Backport vs. compatibility/pruning:** if the exact change exists as a commit in Fedora + dist-git or the upstream project, it is `upstream-backport` (supply `commits`, with + `upstream-status` of `upstreamed` or `upstreamable`). Use an `azl-*` category only when + the change is AZL-specific with no upstream equivalent. +- **Pruning vs. temp-workaround:** removing a dependency we deliberately don't ship is + `azl-pruning`; temporarily working around a dep that *should* exist but hasn't been + imported yet (or any transient workaround waiting on an external fix) is + `azl-temp-workaround`. +- **Flaky vs. unsupported tests:** flaky = the test *could* pass but is intermittent; + unsupported = the test *cannot* run in mock (network/root/hardware). Require evidence of + the limitation before choosing `azl-disable-unsupported-tests`; investigate or ask when + the failure mode is unclear. +- **Compatibility vs. platform-adaptation:** reserve `azl-platform-adaptation` for + architecture-specific (`%ifarch`-style) changes; general toolchain/mock/build-env fixes + are `azl-compatibility`. + +## Step 2 — Set `upstream-status` + +Required whenever `[metadata]` is present. It classifies the overlay's relationship to +upstream — "why are we carrying this?" and "what would it take to drop it?" Pick exactly one: + +| Value | Meaning | +|-------|---------| +| `upstreamed` | Already in Fedora; carried only until AZL bumps past it. | +| `upstreamable` | The patch we carry is itself upstream-shaped (or already in the OSS project but not in Fedora yet); the same diff could be sent upstream and plausibly accepted. Link the upstream PR when you can. | +| `needs-upstream-hook` | AZL-specific change that upstream wouldn't take as-is, but upstream could add a `bcond`/`%if`/config knob so we could drop the overlay. | +| `inapplicable` | Permanent AZL-only deviation with no upstream story (branding, deliberate pruning, enterprise policy). | +| `unknown` | Not yet assessed. Prefer a definite value; reviewers should push back on `unknown` before approving. | + +On an `upstream-backport` overlay only `upstreamed` and `upstreamable` are allowed — any +other value is a validation error. + +**`upstreamable` vs. `needs-upstream-hook`:** `upstreamable` means the patch we carry is +itself upstream-shaped (send the same diff upstream); `needs-upstream-hook` means the +change is AZL-specific and would *not* be accepted as-is, but upstream could add a hook +that makes patching unnecessary. + +## Step 3 — Add provenance (`commits`, `bugs`) + +- `commits` — list of `{ url = "..." }` tables pointing at upstream commits (absolute + http(s) URLs). **Required for `upstream-backport`**; optional elsewhere but valuable + whenever a change traces to a specific commit. For one logical change spanning several + commits, list them all. Verify each SHA actually exists upstream before recording it — a + discovered-and-verified URL is not "inventing" metadata; an unverified guess is. +- `bugs` — list of `{ url = "..." }` tables referencing tracker entries. Never fabricate one. + +## Step 4 — Write the metadata (TOML forms) + +**Prefer the per-file layout (an overlay file loaded via `overlay-files`) for all new +work — even a component with a single overlay.** It keeps `category`/`commits`/`bugs` on +their own lines (no inline-table one-line limit) and means a change never has to be +reshuffled when it grows a second overlay. One top-level `[metadata]` table applies to +every `[[overlays]]` entry in the file. + +Multi-overlay change (one logical change, several overlays): + +```toml +# One logical change: drop the devel sub-package AZL does not ship. +[metadata] +category = "azl-pruning" +upstream-status = "inapplicable" + +[[overlays]] +description = "Remove the devel sub-package — AZL ships no -devel for this component" +type = "spec-remove-subpackage" +package = "devel" + +[[overlays]] +description = "Drop the BuildRequires only the devel sub-package needed" +type = "spec-remove-tag" +tag = "BuildRequires" +value = "some-devel-only-dep" +``` + +### Inline forms + +Use inline `metadata` only when the component is already inline and you are not +restructuring it. A single-line inline table must fit on one line (no lists), so it is +limited to one or two scalar fields: + +```toml +[[components.rpm.overlays]] +description = "Customize RPM vendor to Azure Linux" +type = "spec-search-replace" +regex = "RPM_VENDOR=redhat" +replacement = "RPM_VENDOR=azurelinux" +metadata = { category = "azl-branding-policy", upstream-status = "inapplicable" } +``` + +Use the sub-table form whenever you need a list (`commits`, `bugs`) or more than a couple +of fields: + +```toml +[[components.xclock.overlays]] +description = "Pass --force to autoreconf so the build survives newer autotools" +type = "spec-search-replace" +regex = "autoreconf -i" +replacement = "autoreconf -fi" + +[components.xclock.overlays.metadata] +category = "upstream-backport" +upstream-status = "upstreamed" +commits = [{ url = "https://src.fedoraproject.org/rpms/xclock/c/1e407488" }] +``` + +## Step 5 — Verify + +Adding or editing metadata must be a no-op on the rendered spec. Re-render and confirm +there is no diff: + +```sh +azldev comp render -p +git diff {{ .RenderedSpecsDir }}/ +``` + +If you also moved overlays into files, prove the apply order was preserved with +`azldev comp diff-sources -p ` before and after — any difference means an overlay +changed or the sequence shifted. Metadata-only edits need no rebuild or upstream commit refresh; if +you also changed an overlay's behavior, finalize with the `azldev-refresh-upstream-commit` skill. + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/overlays.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/overlays.md.tmpl new file mode 100644 index 000000000..2235eb559 --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/overlays.md.tmpl @@ -0,0 +1,161 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Working with overlays + +Overlays are **semantic patches** applied to a component's RPM spec and loose +source files at render time. They let you make targeted changes to an upstream +spec without forking it. Prefer an overlay over hand-editing a rendered spec: +overlays are re-applied on every render, so a manual edit to a rendered spec is +overwritten. + +## The inner loop + +Overlays live in the component's TOML config — inline `[[components..overlays]]` +entries, or per-file overlay documents referenced by the component's `overlay-files` +glob. They apply **in order** and are **non-atomic**: if one fails part-way, the +overlays before it stay applied. + +1. Add or edit the overlay in the component config. +2. Re-render and inspect the result: + + ```sh + azldev comp render -p + ``` + + Read the rendered spec (under `{{ .RenderedSpecsDir }}/`) to confirm the change + landed where you intended, and iterate until it is correct. +3. Finalize the upstream commit and changelog with the normal end-of-work refresh (see the + `azldev-refresh-upstream-commit` skill): refresh the generated config, commit, then re-render and amend. + +Config errors reference the offending overlay by its `description`, so give every +overlay a short, specific `description`. + +## Diagnose common failures + +Start with `azldev comp diff-sources -p ` to see the exact overlay effect. +Use separate pre/post `prep-sources` directories only when you need persistent trees +for deeper inspection. + +| Symptom | Likely cause and fix | +| --- | --- | +| `spec-add-tag`: tag already exists | Upstream already has the tag. Use `spec-set-tag`, or `spec-update-tag` when its prior existence is an invariant. | +| `spec-search-replace`: no match | Inspect the current upstream line, check TOML regex quoting, and narrow the expression to the actual section/package. | +| Section or file not found | Inspect the upstream spec/source names; upstream may have renamed or removed the target. | +| Overlay applies but output/build is wrong | Inspect `diff-sources` for an over-broad match, malformed replacement, or a dependency/file change the overlay omitted. | + +## Choosing an overlay type + +Match the change to the narrowest overlay type. Required fields are enforced when +the config loads, so a missing field fails fast rather than at apply time. + +### Spec overlays (structured `.spec` edits) + +| Type | Use for | Required | +| --- | --- | --- | +| `spec-add-tag` | add a tag; fails if it already exists | `tag`, `value` | +| `spec-insert-tag` | add a tag next to its family (e.g. after the last `Source*`) | `tag`, `value` | +| `spec-set-tag` | set a tag, replacing it if present or adding it if not | `tag`, `value` | +| `spec-update-tag` | change an existing tag; fails if it is missing | `tag`, `value` | +| `spec-remove-tag` | delete tag instances; without `value`, deletes every instance | `tag` | +| `spec-prepend-lines` | insert lines at the top of a section (or the whole file) | `lines` | +| `spec-append-lines` | insert lines at the end of a section (or the whole file) | `lines` | +| `spec-search-replace` | regex replace within a section (or the whole spec) | `regex` | +| `spec-remove-section` | delete a whole section | `section` | +| `spec-remove-subpackage` | delete every section of a sub-package | `package` | +| `patch-add` | add a `.patch` file and register it in the spec | `source` | +| `patch-remove` | remove a patch and its spec references | `file` | + +### File overlays (loose non-spec files; never `.spec`) + +| Type | Use for | Required | +| --- | --- | --- | +| `file-prepend-lines` | prepend lines to a file | `file`, `lines` | +| `file-search-replace` | regex replace in a file | `file`, `regex` | +| `file-add` | copy in a new file; fails if it already exists | `file`, `source` | +| `file-remove` | delete a file | `file` | +| `file-rename` | rename a file in place | `file`, `replacement` | + +## Rules that trip people up + +- **`spec-remove-tag` without `value` removes every instance** of the named tag. + To remove one dependency, set both `tag` and the exact `value` to match: + + ```toml + [[components.mypackage.overlays]] + description = "Remove an unavailable build dependency" + type = "spec-remove-tag" + tag = "BuildRequires" + value = "unwanted-package" + ``` +- **`section` is optional only** for `spec-prepend-lines`, `spec-append-lines`, and + `spec-search-replace` (omit it to target the whole spec). It is **required** for + `spec-remove-section`. +- **`package` needs `section`** on the whole-file-capable overlays — a sub-package is + a sub-qualifier of a section. `spec-remove-subpackage` is the exception: it takes + `package` and rejects `section`. +- **`replacement` is literal** — `$1`-style capture-group references are not expanded; + omit it to delete matched text. +- **Quote `regex` as a TOML literal string** — write `regex = '\.so$'`, not + `regex = "\.so$"`. A basic (double-quoted) TOML string interprets backslash escapes, so + `\s`, `\.`, `\d` and friends are mangled before the regex engine ever sees them; single + quotes keep the pattern verbatim. +- **Anchor regex overlays to whole lines, and prefer macro toggles.** When + `spec-search-replace` is unavoidable, anchor the full line (for example, + `regex = '^%setup -q$'`) instead of matching a fragment, and combine several + near-identical patterns into one rather than stacking brittle overlays. If the + upstream spec already exposes a conditional such as `%if 0%{?rhel}` / + `%if 0%{?fedora}` or a definable macro, set that macro instead of rewriting the + line with regex; the explicit toggle survives upstream changes more reliably. +- **`spec-search-replace` matches one line at a time** — the pattern is applied to each + spec line independently, so it can never span a newline and `(?s)`/DOTALL does nothing. + For a multi-line change use a structured spec overlay (`spec-remove-section`, + `spec-prepend-lines`/`spec-append-lines`, etc.). `file-search-replace` is different: it + matches against the whole file, so multi-line patterns (and `(?s)`) work there. +- **`file` is a glob** (`**` supported) for the multi-file file overlays; for `file-add` + and `file-rename` it is a single name, and `file-rename`'s `replacement` is a + filename only (not a path). +- **`source` paths are relative** to the config that declares the overlay — the overlay + file when loaded via `overlay-files`, otherwise the component config. +- **`file-add` lands beside the spec**, in the dist-git sources root — not inside the + extracted upstream tree. Adding a file there does not make the build use it; wire it in + with a `SourceN` tag plus `%prep`/`%install` steps, or use `patch-add` to change tracked + sources. +- **Don't rename the `Name:` tag** with `spec-update-tag`/`spec-set-tag`. `%{name}` feeds + `Source*` URLs, `%setup -n`, and `%files` paths, so renaming it silently breaks those + references. Keep the spec `Name` aligned with the component instead. +- To add a real `.patch` file (rather than an inline edit), use `patch-add`; it copies + the `source` into the component sources and registers a `PatchN` tag or `%patchlist` + entry. + +## Document intent with `metadata` + +Give non-trivial overlays a `metadata` table. It is documentation only and does +not change rendered output. It records *why* the overlay exists and *when* it +can be dropped. Every metadata block +requires `category`; pick the narrowest of: + +`upstream-backport`, `azl-pruning`, `azl-compatibility`, `azl-temp-workaround`, +`azl-branding-policy`, `azl-disable-flaky-tests`, `azl-disable-unsupported-tests`, +`azl-security-compliance`, `azl-release-management`, `azl-platform-adaptation`. + +It also requires `upstream-status`: `upstreamed`, `upstreamable`, +`needs-upstream-hook`, `inapplicable`, or `unknown`. Add `commits` and `bugs` as +`{ url = "https://..." }` entries where they apply. `commits` is required for +`upstream-backport`, whose status must be `upstreamed` or `upstreamable`. When several +overlays share one provenance, put them in a per-file overlay document (`overlay-files`) +with a single file-level `[metadata]`. + +For how to choose the right `category` and `upstream-status`, disambiguation tips, and +the TOML forms, read the `azldev-overlay-metadata` skill. + +## Full reference + +The tables above are the working subset. For the exhaustive field rules, metadata +constraints, and the per-file overlay format, generate the machine-readable schema +with `azldev config generate-schema` (see the `ComponentOverlay` definition), or read +azldev's overlays configuration reference. + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl new file mode 100644 index 000000000..ecd6162d5 --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl @@ -0,0 +1,76 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Refresh component upstream commits + +`azldev comp refresh-upstream-commit` (`comp` is an alias for `component`) resolves one or more +components at the distro snapshot and writes normal component TOML under +`{{ .UpstreamCommitsDir }}/`. + +The project config must include `{{ .UpstreamCommitsDir }}/*.toml` **before** +the component-specific TOML configuration so subsequent commands load the +generated commit pins and the component definition supplies the remaining +`spec` fields. If `--upstream-commits-dir` selects another directory, change +the include pattern to match while preserving that ordering. + +The command uses the fully merged component configuration. It resolves and +writes a pin only when the selected component's effective `spec.type` is +`upstream`; for other source types, it removes any existing generated pin +without contacting an upstream provider. +It loads configuration permissively so a stale generated pin cannot block +cleanup after a component is removed or converted to another source type. +Other configuration validation failures are still reported as warnings. + +## When to run `refresh-upstream-commit` + +| Situation | Run `refresh-upstream-commit`? | +| --- | --- | +| Adding a new upstream component | **Yes** — first, to generate its upstream commit TOML | +| Finalizing overlay, build config, or metadata changes | No — these do not affect the resolved commit | +| Changing source resolution (commit pin, upstream distro/version, or snapshot) | **Yes** — also mid-workflow (see below) | +| Iterating on overlays / build config / metadata | No — `render` alone is enough while iterating | +| Just reading or building existing components | No | + +Refresh a single component with `-p `. Use `-a` (all components) only for +coordinated mass refreshes (e.g. a new distro snapshot) or when investigating +commit drift across many components — it is slow. For day-to-day work use `-p`. +Add `-O json` for machine-readable output when debugging. + +## Finalizing component changes + +For edits that do not change source resolution, render and commit the result. +The generated upstream commit TOML does not change: + +```sh +azldev comp render -p +git add \ + {{ .RenderedSpecsDir }}/// +git commit -m "fix(): ..." +``` + +Synthetic dist-git history is derived from committed upstream-commit TOML changes. Changes +that do not refresh the upstream commit do not add a synthetic history entry or +trigger an automatic release bump. + +## Changing source resolution + +A source-resolution change follows the same rule, using one commit followed by a +post-render amend: + +1. Change the upstream distro/version or snapshot, then `azldev comp refresh-upstream-commit -p `; sanity-check `{{ .UpstreamCommitsDir }}/.toml`. +2. `azldev comp render -p ` — the spec body now tracks the newly resolved source. `%changelog` / `Release:` still reflect the previous source; that is expected until you commit. +3. Iterate on overlays / patches / build config as the new source requires, re-rendering after each change. Re-run `refresh-upstream-commit` only if you change a source-resolution input again. +4. Stage and commit all component inputs changed above with the refreshed TOML and rendered output: + + ```sh + git add \ + {{ .UpstreamCommitsDir }}/.toml \ + {{ .RenderedSpecsDir }}/// + git commit -m "feat(): refresh upstream source" + ``` +5. `azldev comp render -p ` — `%changelog` / `Release:` now reflect the new commit TOML. +6. `git add {{ .RenderedSpecsDir }}///`, then `git commit --amend --no-edit` so the source change and rendered output land together. + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/remove-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/remove-component.md.tmpl new file mode 100644 index 000000000..a7d36e2a6 --- /dev/null +++ b/internal/app/azldev/agentskill/content/withoutlockfile/remove-component.md.tmpl @@ -0,0 +1,79 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Remove a component + +There is **no `azldev` command to remove a component** — it is a manual edit. A +deletion-only change needs no package build, but changes to dependents, package/image +configuration, or other built inputs must follow the repository's normal validation +requirements. The steps below delete the component's definition, generated commit config, and rendered +spec, then clean up references. + +## Before you start + +Confirm the component exists and inspect its resolved configuration: + +```sh +azldev comp list -p -q -O json +``` + +Check for **reverse dependencies** first: if other components `BuildRequires` or +`Requires` this one, removing it breaks their builds. Search the tree for the +component name. Identify its binary subpackage names from the rendered output under +`{{ .RenderedSpecsDir }}///`, then search for those names too. + +## Steps + +1. **Remove the definition.** Delete the component's dedicated + `/.comp.toml` directory, or remove its inline `[components.]` + entry from whichever included TOML defines it. +2. **Remove publish / package config references.** If your project configures + publish channels or package groups, drop any references to the component or to + its binary subpackages. Publishing is component-scoped; per-binary exceptions + are binary-RPM-scoped — search for both. +3. **Remove the generated upstream commit config.** There is no targeted azldev command for this: + + ```sh + rm {{ .UpstreamCommitsDir }}/.toml + ``` + +4. **Remove the rendered spec.** Let azldev prune orphaned spec directories: + + ```sh + azldev comp render -a --clean-stale + ``` + + `--clean-stale` (only valid with `-a`) removes rendered-spec directories that no + longer correspond to a configured component. It re-renders everything, so it is + slow; for a targeted removal you can instead delete the component's rendered + spec under `{{ .RenderedSpecsDir }}/` by hand. +5. **Check other references.** Grep for the component (and its binary names) in + image definitions and any `*.kiwi` / package-list files, and remove or replace + them — an image that installs a now-removed package will not build. + +## Verify + +```sh +azldev comp list -p -q -O json # should report the component is not found +``` + +Also confirm the generated commit config and rendered spec directory are gone and that no image or +package configuration still references the component. + +If the removal required changes to dependents or package/image configuration, build +and test the affected outputs according to the repository's normal validation policy. + +## Notes + +- **Dependents with manual release.** If removing this component forces a change + in a *dependent* (e.g. dropping a `BuildRequires`), and that dependent sets + `release.calculation = "manual"`, bump its release counter yourself in the same + change. Components with automatic release calculation (`auto`, `autorelease`, + `static`) are handled by the normal commit/render/amend cycle. +- **Remove exclusive dependencies together.** If a package is only needed by the + component you are dropping, remove it in the same change to keep the tree + consistent. + +Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/doc.go b/internal/app/azldev/agentskill/doc.go index 18e3816c2..890616c4c 100644 --- a/internal/app/azldev/agentskill/doc.go +++ b/internal/app/azldev/agentskill/doc.go @@ -55,8 +55,18 @@ // - Version — the azldev version stamped into every file. // - TopLevelCommands — generated from the Cobra command tree, so the overview skill's // command list never goes stale. -// - Bindings — repo-specific paths (LockDir, RenderedSpecsDir, WorkDir) read from -// the target azldev.toml, degrading to azldev's defaults when no config is present. +// - Bindings — repo-specific paths (LockDir, UpstreamCommitsDir, RenderedSpecsDir, +// WorkDir) read from the target azldev.toml, degrading to azldev's defaults when no +// config is present. +// +// # Modes +// +// A [Catalog] selects the content for one of azldev's two modes. The default mode +// documents the lock-file workflow; the mode selected by the global +// '--without-lockfile' flag documents the generated upstream-commit workflow. Skills, +// instruction files, and emitted layout are shared; templates under +// 'content/withoutlockfile' replace the same-named default template, and a small +// registry replaces the skills whose subject differs. // // # Outputs (three sinks, one registry) // diff --git a/internal/app/azldev/app.go b/internal/app/azldev/app.go index f387880dd..6f87ebe45 100644 --- a/internal/app/azldev/app.go +++ b/internal/app/azldev/app.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "path/filepath" + "strconv" "strings" "time" @@ -49,8 +50,10 @@ type App struct { reportFormat ReportFormat disableDefaultConfig bool permissiveConfigParsing bool + commandPermissiveConfig bool configFiles []string colorMode ColorMode + withoutLockfile bool // Root command for the CLI. cmd cobra.Command @@ -135,7 +138,7 @@ lives), or use -C to point to one.`, env.SetAcceptAllPrompts(app.acceptAllPrompts) env.SetColorMode(app.colorMode) env.SetNetworkRetries(app.networkRetries) - env.SetPermissiveConfigParsing(app.permissiveConfigParsing) + env.SetPermissiveConfigParsing(app.permissiveConfigEnabled()) return nil }, @@ -184,6 +187,8 @@ func (app *App) registerGlobalFlags() { "output colorization mode {always, auto, never}") app.cmd.PersistentFlags().BoolVar(&app.permissiveConfigParsing, "permissive-config", false, "do not fail on unknown fields in TOML config files") + app.cmd.PersistentFlags().BoolVar(&app.withoutLockfile, "without-lockfile", false, + "preview: track resolved upstream commits in generated config instead of lock files") } // addAdvancedCommandHint embeds a hint about the hidden "advanced" command group @@ -257,7 +262,8 @@ func (a *App) Execute(args []string) int { // We tried to do this with cobra first, but it was too difficult to get // the "right thing" to happen. // - a.handParseConfigFlags(args) + a.PreParseGlobalFlags(args) + a.commandPermissiveConfig = a.commandRequestsPermissiveConfig(args) envOptions := a.initializeEnvOptions() @@ -361,6 +367,25 @@ func (a *App) Execute(args []string) int { return a.dispatchToCommand(env, args) } +// commandRequestsPermissiveConfig reports whether the command selected by args asked +// for permissive configuration loading via [CommandAnnotationPermissiveConfig]. +func (a *App) commandRequestsPermissiveConfig(args []string) bool { + cmd, _, err := a.cmd.Find(args) + if err != nil { + return false + } + + _, permissive := cmd.Annotations[CommandAnnotationPermissiveConfig] + + return permissive +} + +// permissiveConfigEnabled reports whether configuration should be loaded permissively, +// either because the user asked for it or because the selected command requires it. +func (a *App) permissiveConfigEnabled() bool { + return a.permissiveConfigParsing || a.commandPermissiveConfig +} + func (*App) setCmdFactory(envOptions *EnvOptions) error { cmdFactory, err := DefaultCmdFactory(envOptions.DryRunnable, envOptions.EventListener) if err != nil { @@ -397,6 +422,7 @@ func (a *App) initializeEnvOptions() *EnvOptions { envOptions.Interfaces.FileSystemFactory = a.fsFactory envOptions.Interfaces.OSEnvFactory = a.osEnvFactory envOptions.DryRunnable = NewAppDryRunnable(a.dryRun) + envOptions.WithoutLockfile = a.withoutLockfile return &envOptions } @@ -458,6 +484,68 @@ func setEventListener(stdioLogger *slog.Logger, quiet, verbose bool, envOptions return nil } +// PreParseGlobalFlags hand-parses the global flags that must be known before commands +// are registered and configuration is loaded. It is safe to call more than once with the +// same arguments; each call fully recomputes the pre-parsed state. +// +// Command registration is mode-sensitive, so the CLI entry point calls this before it +// registers commands; [App.Execute] calls it again so that an App executed directly +// (e.g. from a test) behaves identically. +func (a *App) PreParseGlobalFlags(args []string) { + // Reset accumulating state so repeated calls are idempotent. + a.configFiles = nil + + a.withoutLockfile = parseWithoutLockfileFlag(args) + a.handParseConfigFlags(args) +} + +// WithoutLockfile reports whether the preview lock-file-free mode was requested via the +// global '--without-lockfile' flag. Valid only after [App.PreParseGlobalFlags] has run. +func (a *App) WithoutLockfile() bool { + return a.withoutLockfile +} + +// withoutLockfileFlagName is the global flag that selects lock-file-free mode. +const withoutLockfileFlagName = "--without-lockfile" + +// parseWithoutLockfileFlag hand-parses the global '--without-lockfile' boolean flag. +// +// Only exact '--without-lockfile' and '--without-lockfile=' tokens are +// recognized, and scanning stops at the '--' terminator so that positional text +// (for example a mock command line) is never mistaken for the flag. An invalid or +// missing value is treated the same way cobra treats it: '--without-lockfile' alone +// enables the mode, and an unparseable '=' leaves the mode disabled so the +// final cobra parse reports the error. +func parseWithoutLockfileFlag(args []string) bool { + withoutLockfile := false + + for _, arg := range args { + if arg == "--" { + break + } + + if arg == withoutLockfileFlagName { + withoutLockfile = true + + continue + } + + value, found := strings.CutPrefix(arg, withoutLockfileFlagName+"=") + if !found { + continue + } + + parsed, err := strconv.ParseBool(value) + if err != nil { + return false + } + + withoutLockfile = parsed + } + + return withoutLockfile +} + // Hand-parses a few critical configuration flags from the command line -- just enough to // find the project and load configuration so we can properly use cobra facilities to // parse the full command line. @@ -536,7 +624,8 @@ func (a *App) findAndLoadConfig(tempDirPath string, extraConfigFiles []string) ( a.disableDefaultConfig, tempDirPath, extraConfigFiles, - a.permissiveConfigParsing, + a.permissiveConfigEnabled(), + a.withoutLockfile, ) if err != nil { return projectDir, config, fmt.Errorf("failed to load project configuration:\n%w", err) diff --git a/internal/app/azldev/app_test.go b/internal/app/azldev/app_test.go index ddc2d8ced..0531f2db9 100644 --- a/internal/app/azldev/app_test.go +++ b/internal/app/azldev/app_test.go @@ -193,3 +193,100 @@ func TestApp_PermissiveConfigOption_DefaultFalse(t *testing.T) { assert.Zero(t, result) assert.True(t, ran) } + +func TestApp_WithoutLockfileOption(t *testing.T) { + testCases := []struct { + name string + args []string + expected bool + }{ + {name: "absent", args: []string{"test-cmd"}, expected: false}, + {name: "bare", args: []string{"--without-lockfile", "test-cmd"}, expected: true}, + {name: "explicit true", args: []string{"--without-lockfile=true", "test-cmd"}, expected: true}, + {name: "explicit false", args: []string{"--without-lockfile=false", "test-cmd"}, expected: false}, + {name: "after command", args: []string{"test-cmd", "--without-lockfile"}, expected: true}, + {name: "positional lookalike", args: []string{"test-cmd", "--", "--without-lockfile"}, expected: false}, + {name: "prefix lookalike", args: []string{"test-cmd", "--without-lockfiles"}, expected: false}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + app := createTestApp(t) + + app.PreParseGlobalFlags(testCase.args) + + assert.Equal(t, testCase.expected, app.WithoutLockfile()) + }) + } +} + +func TestApp_WithoutLockfileOption_ReachesEnv(t *testing.T) { + testCases := []struct { + name string + args []string + expected bool + }{ + {name: "default", args: []string{"test-cmd"}, expected: false}, + {name: "enabled", args: []string{"--without-lockfile", "test-cmd"}, expected: true}, + {name: "disabled", args: []string{"--without-lockfile=false", "test-cmd"}, expected: false}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + app := createTestApp(t) + + ran := false + cmd := &cobra.Command{ + Use: "test-cmd", + RunE: func(cmd *cobra.Command, _ []string) error { + env, err := azldev.GetEnvFromCommand(cmd) + require.NoError(t, err) + + assert.Equal(t, testCase.expected, env.WithoutLockfile()) + + ran = true + + return nil + }, + } + + app.AddTopLevelCommand(cmd) + + assert.Zero(t, app.Execute(testCase.args)) + assert.True(t, ran) + }) + } +} + +// PreParseGlobalFlags is called both by the CLI entry point (before command +// registration) and by Execute; repeated calls must not accumulate state. +func TestApp_PreParseGlobalFlags_Idempotent(t *testing.T) { + app := createTestApp(t) + args := []string{"--without-lockfile", "--config-file", "extra.toml", "test-cmd"} + + ran := false + cmd := &cobra.Command{ + Use: "test-cmd", + RunE: func(cmd *cobra.Command, _ []string) error { + env, err := azldev.GetEnvFromCommand(cmd) + require.NoError(t, err) + + assert.True(t, env.WithoutLockfile()) + + ran = true + + return nil + }, + } + + app.AddTopLevelCommand(cmd) + app.PreParseGlobalFlags(args) + app.PreParseGlobalFlags(args) + + assert.True(t, app.WithoutLockfile()) + + // Execute pre-parses the same arguments a third time; a command that only + // accumulated config files would fail to load its (nonexistent) extra file. + assert.Zero(t, app.Execute(args)) + assert.True(t, ran) +} diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index 60aed20f0..0bfdbebd5 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -78,11 +78,11 @@ type ComponentBuildResults struct { RPMs []RPMResult `json:"rpms" table:"-"` } -func buildOnAppInit(_ *azldev.App, parent *cobra.Command) { - parent.AddCommand(NewBuildCmd()) +func buildOnAppInit(app *azldev.App, parent *cobra.Command) { + parent.AddCommand(NewBuildCmd(cmdOptionsForApp(app)...)) } -func NewBuildCmd() *cobra.Command { +func NewBuildCmd(opts ...CmdOption) *cobra.Command { // Fill out options defaults. options := &ComponentBuildOptions{ BuildEnvPolicy: BuildEnvPreserveOnFailure, @@ -127,7 +127,7 @@ builds can consume.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) cmd.Flags().BoolVarP(&options.ContinueOnError, "continue-on-error", "k", false, "Continue building when some components fail") cmd.Flags().BoolVar(&options.NoCheck, "no-check", false, "Skip package %check tests") @@ -269,10 +269,7 @@ func buildComponent( var preparerOpts []sources.PreparerOption if !options.WithoutGitRepo { - preparerOpts = append(preparerOpts, - sources.WithGitRepo(env, env.LockReader(), distro.Version.ReleaseVer), - sources.WithDirtyDetection(), - ) + preparerOpts = append(preparerOpts, gitRepoPreparerOptions(env, distro)...) } preparerOpts = append(preparerOpts, diff --git a/internal/app/azldev/cmds/component/changed.go b/internal/app/azldev/cmds/component/changed.go index c74cd49e3..40e947df5 100644 --- a/internal/app/azldev/cmds/component/changed.go +++ b/internal/app/azldev/cmds/component/changed.go @@ -32,18 +32,36 @@ type ChangedComponentOptions struct { IncludeUnchanged bool } -func changedOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewChangedCmd()) +func changedOnAppInit(app *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewChangedCmd(cmdOptionsForApp(app)...)) } -// NewChangedCmd constructs a [cobra.Command] for the "component changed" CLI subcommand. -func NewChangedCmd() *cobra.Command { - options := &ChangedComponentOptions{} +// changedCommandLong returns the long help for 'component changed', which +// describes the comparison performed by the active mode. +func changedCommandLong(options cmdOptions) string { + if options.withoutLockfile { + return `Load the project configuration independently at two git refs and compare the +resolved component build inputs. Normal project TOML parsing is used at each +ref, including recursive includes, config merging, component defaults, and +generated upstream-commit TOMLs included by the project. + +For each selected component, the command reports whether the build inputs +changed: normalized component configuration, upstream commit or local +spec-directory contents, overlay source filenames and contents, and effective +distro release version. + +Documentation, publishing, test-selection, scheduling-hint, snapshot-time, and +checkout-path-only fields do not trigger a component change. The sourcesChange +field separately compares the raw committed sources manifests at the rendered +spec directories configured by each ref. All-component scans use the union of +components in both refs, so added and deleted components are reported without +consulting the current checkout. - cmd := &cobra.Command{ - Use: "changed", - Short: "Detect which components changed between two git refs", - Long: `Compare component lock files and rendered sources between two git refs to +This is useful for CI/CD pipelines to determine which components need to be +rebuilt or have their lookaside tarballs re-uploaded after a PR merge.` + } + + return `Compare component lock files and rendered sources between two git refs to determine which components changed. For each component, reports whether its input fingerprint changed (any change) and whether its rendered sources file changed (sources change). @@ -61,7 +79,18 @@ Note: component selection and directory paths (lock-dir, rendered-specs-dir) are resolved from the current checkout's configuration, not from the compared refs. For accurate results, run this command from a checkout that matches the --to ref (e.g., after merging a PR). Components not in the current config are -detected via lock file presence in the compared refs when using -a.`, +detected via lock file presence in the compared refs when using -a.` +} + +// NewChangedCmd constructs a [cobra.Command] for the "component changed" CLI subcommand. +func NewChangedCmd(opts ...CmdOption) *cobra.Command { + options := &ChangedComponentOptions{} + cmdOptions := newCmdOptions(opts...) + + cmd := &cobra.Command{ + Use: "changed", + Short: "Detect which components changed between two git refs", + Long: changedCommandLong(cmdOptions), Example: ` # Show changed components between a branch and HEAD azldev component changed --from main -a @@ -81,7 +110,7 @@ detected via lock file presence in the compared refs when using -a.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions) cmd.Flags().StringVar(&options.From, "from", "", "Git ref to compare from (required)") cmd.Flags().StringVar(&options.To, "to", "HEAD", "Git ref to compare to") @@ -92,7 +121,9 @@ detected via lock file presence in the compared refs when using -a.`, // Hide inherited flag -- this command always skips lock validation since // it inspects historical locks at arbitrary refs. - _ = cmd.Flags().MarkHidden("skip-lock-validation") + if !cmdOptions.withoutLockfile { + _ = cmd.Flags().MarkHidden("skip-lock-validation") + } azldev.ExportAsReadOnlyMCPTool(cmd) @@ -114,10 +145,24 @@ const ( changeTypeDeleted = "deleted" ) -// ChangedComponents compares component lock files and rendered sources between -// two git refs to determine which changed. +// ChangedComponents determines which components changed between two git refs, +// using the comparison that matches the active mode: stored lock file +// fingerprints by default, or configuration resolved at each ref when the global +// '--without-lockfile' flag is set. func ChangedComponents( env *azldev.Env, options *ChangedComponentOptions, +) ([]ChangedResult, error) { + if env.WithoutLockfile() { + return changedComponentsFromProjectConfigs(env, options) + } + + return changedComponentsFromLocks(env, options) +} + +// changedComponentsFromLocks compares component lock files and rendered sources +// between two git refs to determine which changed. +func changedComponentsFromLocks( + env *azldev.Env, options *ChangedComponentOptions, ) ([]ChangedResult, error) { // Changed compares lock files between git refs — skip validation since // the current working-tree locks may legitimately be stale. @@ -210,19 +255,28 @@ type changedContext struct { integrityViolations []string } -// newChangedContext opens the project repository and resolves paths. -func newChangedContext(env *azldev.Env) (*changedContext, error) { +// openChangedRepo opens the project's git repository and returns it together with +// its worktree root. +func openChangedRepo(env *azldev.Env) (*gogit.Repository, string, error) { repo, err := git.OpenProjectRepo(env.ProjectDir()) if err != nil { - return nil, fmt.Errorf("opening project repository:\n%w", err) + return nil, "", fmt.Errorf("opening project repository:\n%w", err) } worktree, err := repo.Worktree() if err != nil { - return nil, fmt.Errorf("getting project worktree:\n%w", err) + return nil, "", fmt.Errorf("getting project worktree:\n%w", err) } - repoRoot := worktree.Filesystem.Root() + return repo, worktree.Filesystem.Root(), nil +} + +// newChangedContext opens the project repository and resolves paths. +func newChangedContext(env *azldev.Env) (*changedContext, error) { + repo, repoRoot, err := openChangedRepo(env) + if err != nil { + return nil, err + } lockRelDir, err := repoRelPath(repoRoot, env.Config().Project.LockDir) if err != nil { diff --git a/internal/app/azldev/cmds/component/changed_test.go b/internal/app/azldev/cmds/component/changed_test.go index 8a0be07a0..0e0952ea6 100644 --- a/internal/app/azldev/cmds/component/changed_test.go +++ b/internal/app/azldev/cmds/component/changed_test.go @@ -59,3 +59,12 @@ func TestChangedCmd_NoComponents(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "component not found") } + +func TestNewChangedCmd_LongDescriptionByMode(t *testing.T) { + defaultCmd := componentcmds.NewChangedCmd() + assert.Contains(t, defaultCmd.Long, "Compare component lock files") + + withoutLockfileCmd := componentcmds.NewChangedCmd(componentcmds.WithoutLockfileFlags()) + assert.Contains(t, withoutLockfileCmd.Long, "Load the project configuration independently") + assert.NotContains(t, withoutLockfileCmd.Long, "lock file") +} diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go new file mode 100644 index 000000000..e46e0e46a --- /dev/null +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -0,0 +1,648 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" +) + +// This file implements 'component changed' for lock-file-free mode. Without lock +// files there are no stored fingerprints to compare, so the project configuration +// is loaded independently at both refs and the resolved component build inputs are +// compared directly. The git plumbing helpers are shared with the default +// implementation in changed.go. + +// changedComponentsFromProjectConfigs loads and compares resolved project +// components and rendered sources between two git refs. It is the lock-file-free +// implementation of 'component changed'. +func changedComponentsFromProjectConfigs( + env *azldev.Env, options *ChangedComponentOptions, +) ([]ChangedResult, error) { + repo, repoRoot, err := openChangedRepo(env) + if err != nil { + return nil, err + } + + fromHash, err := resolveCommitHash(repo, options.From) + if err != nil { + return nil, fmt.Errorf("resolving --from ref %#q:\n%w", options.From, err) + } + + toHash, err := resolveCommitHash(repo, options.To) + if err != nil { + return nil, fmt.Errorf("resolving --to ref %#q:\n%w", options.To, err) + } + + projectRelDir, err := repoRelPath(repoRoot, env.ProjectDir()) + if err != nil { + return nil, fmt.Errorf("resolving project directory within repository:\n%w", err) + } + + fromTree, err := resolveTree(repo, fromHash) + if err != nil { + return nil, fmt.Errorf("resolving tree for --from:\n%w", err) + } + + toTree, err := resolveTree(repo, toHash) + if err != nil { + return nil, fmt.Errorf("resolving tree for --to:\n%w", err) + } + + fromProject, err := loadHistoricalProject(env, fromTree, projectRelDir) + if err != nil { + return nil, fmt.Errorf("loading project at --from:\n%w", err) + } + + toProject, err := loadHistoricalProject(env, toTree, projectRelDir) + if err != nil { + return nil, fmt.Errorf("loading project at --to:\n%w", err) + } + + names, err := selectHistoricalComponentNames( + env, &options.ComponentFilter, fromProject, toProject, repoRoot, + ) + if err != nil { + return nil, fmt.Errorf("selecting components:\n%w", err) + } + + return buildHistoricalResults( + names, fromProject, toProject, fromTree, toTree, + options.IncludeUnchanged, options.ComponentFilter.IncludeAllComponents, + ) +} + +const ( + snapshotRepoRoot = "/repo" + snapshotTempDir = "/config-tmp" +) + +type fixedFileSystemFactory struct { + fs opctx.FS +} + +func (factory *fixedFileSystemFactory) FS() opctx.FS { + return factory.fs +} + +// historicalProject contains only data that remains valid after the temporary +// in-memory checkout used to resolve it is discarded. +type historicalProject struct { + components map[string]projectconfig.ComponentConfig + comparisonInputs map[string]componentComparisonInputs + componentGroups map[string][]string + renderedSpecsRelDir string +} + +// componentComparisonInputs mirrors the build-relevant inputs covered by the +// input fingerprint on the main branch. +type componentComparisonInputs struct { + Config projectconfig.ComponentConfig `json:"config"` + SourceIdentity string `json:"sourceIdentity,omitempty"` + OverlaySourceHashes map[string]string `json:"overlaySourceHashes,omitempty"` + ReleaseVer string `json:"releaseVer,omitempty"` +} + +func loadHistoricalProject( + env *azldev.Env, + tree *object.Tree, + projectRelDir string, +) (*historicalProject, error) { + snapshotFS := afero.NewMemMapFs() + + if err := copyTreeToFS(tree, snapshotFS, snapshotRepoRoot); err != nil { + return nil, fmt.Errorf("materializing git tree:\n%w", err) + } + + if err := fileutils.MkdirAll(snapshotFS, snapshotTempDir); err != nil { + return nil, fmt.Errorf("creating temporary config directory:\n%w", err) + } + + projectDir := filepath.Join(snapshotRepoRoot, projectRelDir) + + loadedProjectDir, config, err := projectconfig.LoadProjectConfig( + snapshotFS, + env.OSEnv(), + projectDir, + false, + snapshotTempDir, + nil, + env.PermissiveConfigParsing(), + true, /*withoutLockfile*/ + ) + if err != nil { + return nil, fmt.Errorf("parsing project configuration:\n%w", err) + } + + historicalEnv := azldev.NewEnv(env.Context(), azldev.EnvOptions{ + ProjectDir: loadedProjectDir, + Config: config, + WithoutLockfile: true, + Interfaces: azldev.SystemInterfaces{ + FileSystemFactory: &fixedFileSystemFactory{fs: snapshotFS}, + }, + }) + historicalEnv.SetPermissiveConfigParsing(env.PermissiveConfigParsing()) + + resolver := components.NewResolver(historicalEnv) + + resolvedComponents, err := resolver.FindAllComponents() + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + result := &historicalProject{ + components: make(map[string]projectconfig.ComponentConfig, resolvedComponents.Len()), + comparisonInputs: make(map[string]componentComparisonInputs, resolvedComponents.Len()), + componentGroups: make(map[string][]string, len(config.ComponentGroups)), + } + + if err := populateHistoricalComponents( + result, snapshotFS, historicalEnv, resolvedComponents, + ); err != nil { + return nil, err + } + + if err := populateHistoricalGroups(result, resolver, config); err != nil { + return nil, err + } + + result.renderedSpecsRelDir, err = repoRelPath(snapshotRepoRoot, config.Project.RenderedSpecsDir) + if err != nil { + return nil, fmt.Errorf("resolving rendered specs directory:\n%w", err) + } + + return result, nil +} + +func populateHistoricalGroups( + project *historicalProject, + resolver *components.Resolver, + config *projectconfig.ProjectConfig, +) error { + for groupName := range config.ComponentGroups { + memberNames := make(map[string]bool) + for _, memberName := range config.ComponentGroups[groupName].Components { + memberNames[memberName] = true + } + + group, groupErr := resolver.GetComponentGroupByName(groupName) + if groupErr != nil { + return fmt.Errorf("resolving component group %#q:\n%w", groupName, groupErr) + } + + for _, member := range group.Components { + memberNames[member.ComponentName] = true + } + + project.componentGroups[groupName] = sortedComponentNames(memberNames) + } + + return nil +} + +func populateHistoricalComponents( + project *historicalProject, + fs opctx.FS, + env *azldev.Env, + resolvedComponents *components.ComponentSet, +) error { + for _, component := range resolvedComponents.Components() { + project.components[component.GetName()] = *component.GetConfig() + + inputs, err := buildComponentComparisonInputs(fs, env, component.GetConfig()) + if err != nil { + return fmt.Errorf( + "building change inputs for component %#q:\n%w", + component.GetName(), err, + ) + } + + project.comparisonInputs[component.GetName()] = inputs + } + + return nil +} + +func buildComponentComparisonInputs( + fs opctx.FS, + env *azldev.Env, + component *projectconfig.ComponentConfig, +) (componentComparisonInputs, error) { + inputs := componentComparisonInputs{ + Config: normalizeComponentForComparison(*component), + SourceIdentity: component.EffectiveUpstreamCommit(), + } + + if component.Spec.SourceType == projectconfig.SpecSourceTypeLocal { + identity, err := sourceproviders.ResolveLocalSourceIdentity( + fs, filepath.Dir(component.Spec.Path), + ) + if err != nil { + return inputs, fmt.Errorf("resolving local source identity:\n%w", err) + } + + inputs.SourceIdentity = identity + } + + ref := component.Spec.UpstreamDistro + if ref.Name == "" { + ref = env.Config().Project.DefaultDistro + } + + if ref.Name != "" { + _, distroVersion, err := env.ResolveDistroRef(ref) + if err != nil { + return inputs, fmt.Errorf("resolving distro reference %#q:\n%w", ref.Name, err) + } + + inputs.ReleaseVer = distroVersion.ReleaseVer + } + + for idx, overlay := range component.Overlays { + sourceName := overlay.EffectiveSourceName() + if sourceName == "" { + continue + } + + contentHash, err := fileutils.ComputeFileHash( + fs, fileutils.HashTypeSHA256, overlay.Source, + ) + if err != nil { + return inputs, fmt.Errorf("hashing overlay source %#q:\n%w", overlay.Source, err) + } + + if inputs.OverlaySourceHashes == nil { + inputs.OverlaySourceHashes = make(map[string]string) + } + + inputs.OverlaySourceHashes[strconv.Itoa(idx)] = sourceName + ":" + contentHash + } + + return inputs, nil +} + +// normalizeComponentForComparison removes the same non-build fields that the +// main branch excluded from component input fingerprints. +func normalizeComponentForComparison(component projectconfig.ComponentConfig) projectconfig.ComponentConfig { + normalized := component + normalized.Name = "" + normalized.SourceConfigFile = nil + normalized.RenderedSpecDir = "" + normalized.Spec.Path = "" + normalized.Spec.UpstreamDistro.Snapshot = "" + normalized.Build.Check.SkipReason = "" + normalized.Build.Failure = projectconfig.ComponentBuildFailureConfig{} + normalized.Build.Hints = projectconfig.ComponentBuildHints{} + normalized.OverlayFiles = nil + normalized.Publish = projectconfig.ComponentPublishConfig{} + normalized.Tests = nil + + normalized.Overlays = slices.Clone(component.Overlays) + for idx := range normalized.Overlays { + normalized.Overlays[idx].Description = "" + normalized.Overlays[idx].Source = "" + normalized.Overlays[idx].Metadata = nil + } + + normalized.SourceFiles = slices.Clone(component.SourceFiles) + for idx := range normalized.SourceFiles { + normalized.SourceFiles[idx].Origin.Type = "" + normalized.SourceFiles[idx].Origin.Uri = "" + normalized.SourceFiles[idx].ReplaceReason = "" + } + + if component.Packages != nil { + normalized.Packages = make(map[string]projectconfig.PackageConfig, len(component.Packages)) + for name, pkg := range component.Packages { + pkg.Publish = projectconfig.PackagePublishConfig{} + normalized.Packages[name] = pkg + } + } + + return normalized +} + +func copyTreeToFS(tree *object.Tree, fs opctx.FS, destinationRoot string) error { + files := tree.Files() + + err := files.ForEach(func(file *object.File) error { + content, err := file.Contents() + if err != nil { + return fmt.Errorf("reading %#q:\n%w", file.Name, err) + } + + destinationPath := filepath.Join(destinationRoot, filepath.FromSlash(file.Name)) + if err := fileutils.MkdirAll(fs, filepath.Dir(destinationPath)); err != nil { + return fmt.Errorf("creating parent directory for %#q:\n%w", file.Name, err) + } + + if err := fileutils.WriteFile( + fs, destinationPath, []byte(content), fileperms.PublicFile, + ); err != nil { + return fmt.Errorf("writing %#q:\n%w", file.Name, err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("iterating git tree files:\n%w", err) + } + + return nil +} + +func selectHistoricalComponentNames( + env *azldev.Env, + filter *components.ComponentFilter, + fromProject, toProject *historicalProject, + repoRoot string, +) ([]string, error) { + allNames := make(map[string]bool, len(fromProject.components)+len(toProject.components)) + for name := range fromProject.components { + allNames[name] = true + } + + for name := range toProject.components { + allNames[name] = true + } + + if filter.HasNoCriteria() { + slog.Warn("No component selection options were given, no components will be selected.") + + return []string{}, nil + } + + selected := make(map[string]bool) + + if filter.IncludeAllComponents { + for name := range allNames { + selected[name] = true + } + + return sortedComponentNames(selected), nil + } + + if err := addPatternSelections( + selected, allNames, filter.ComponentNamePatterns, + ); err != nil { + return nil, err + } + + if err := addGroupSelections( + selected, fromProject, toProject, filter.ComponentGroupNames, + ); err != nil { + return nil, err + } + + if err := addSpecPathSelections( + env, selected, fromProject, toProject, repoRoot, filter.SpecPaths, + ); err != nil { + return nil, err + } + + return sortedComponentNames(selected), nil +} + +func addPatternSelections( + selected, allNames map[string]bool, + patterns []string, +) error { + for _, pattern := range patterns { + matched := false + + for name := range allNames { + isMatch, err := filepath.Match(pattern, name) + if err != nil { + return fmt.Errorf("comparing component pattern %#q:\n%w", pattern, err) + } + + if isMatch { + selected[name] = true + matched = true + } + } + + if !matched && !strings.ContainsAny(pattern, "*?[") { + return fmt.Errorf("component not found: %#q", pattern) + } + } + + return nil +} + +func addGroupSelections( + selected map[string]bool, + fromProject, toProject *historicalProject, + groupNames []string, +) error { + for _, groupName := range groupNames { + fromMembers, inFrom := fromProject.componentGroups[groupName] + + toMembers, inTo := toProject.componentGroups[groupName] + if !inFrom && !inTo { + return fmt.Errorf("%w: %#q", components.ErrComponentGroupNotFound, groupName) + } + + for _, name := range append(fromMembers, toMembers...) { + selected[name] = true + } + } + + return nil +} + +func addSpecPathSelections( + env *azldev.Env, + selected map[string]bool, + fromProject, toProject *historicalProject, + repoRoot string, + specPaths []string, +) error { + for _, specPath := range specPaths { + specRelPath, err := projectSpecRepoRelPath(env, repoRoot, specPath) + if err != nil { + return err + } + + snapshotSpecPath := filepath.Join(snapshotRepoRoot, specRelPath) + matched := false + + for _, project := range []*historicalProject{fromProject, toProject} { + for name, component := range project.components { + if filepath.Clean(component.Spec.Path) == snapshotSpecPath { + selected[name] = true + matched = true + } + } + } + + if !matched { + return fmt.Errorf("component not found for spec path %#q", specPath) + } + } + + return nil +} + +func projectSpecRepoRelPath(env *azldev.Env, repoRoot, specPath string) (string, error) { + absolutePath := specPath + if !filepath.IsAbs(absolutePath) { + absolutePath = filepath.Join(env.ProjectDir(), absolutePath) + } + + relativePath, err := repoRelPath(repoRoot, absolutePath) + if err != nil { + return "", fmt.Errorf("resolving spec path %#q:\n%w", specPath, err) + } + + return relativePath, nil +} + +func sortedComponentNames(names map[string]bool) []string { + result := make([]string, 0, len(names)) + for name := range names { + result = append(result, name) + } + + sort.Strings(result) + + return result +} + +func buildHistoricalResults( + names []string, + fromProject, toProject *historicalProject, + fromTree, toTree *object.Tree, + includeUnchanged, includeAllComponents bool, +) ([]ChangedResult, error) { + results := make([]ChangedResult, 0, len(names)) + + for _, name := range names { + result, err := classifyHistoricalComponent( + name, fromProject.comparisonInputs, toProject.comparisonInputs, + ) + if err != nil { + return nil, fmt.Errorf("comparing component %#q:\n%w", name, err) + } + + result.SourcesChange, err = compareHistoricalSources( + fromTree, + toTree, + fromProject.renderedSpecsRelDir, + toProject.renderedSpecsRelDir, + name, + ) + if err != nil { + return nil, fmt.Errorf("comparing sources for %#q:\n%w", name, err) + } + + if includeAllComponents && + !includeUnchanged && + result.ChangeType == changeTypeUnchanged && + !result.SourcesChange { + continue + } + + results = append(results, result) + } + + return results, nil +} + +// classifyHistoricalComponent compares the serialized build-relevant inputs for a +// component at two refs. +func classifyHistoricalComponent( + name string, + fromComponents, toComponents map[string]componentComparisonInputs, +) (ChangedResult, error) { + result := ChangedResult{ + Component: name, + ChangeType: changeTypeUnchanged, + } + + fromComponent, inFrom := fromComponents[name] + toComponent, inTo := toComponents[name] + + switch { + case !inFrom && !inTo: + result.ChangeType = changeTypeUnchanged + case !inFrom: + result.ChangeType = changeTypeAdded + case !inTo: + result.ChangeType = changeTypeDeleted + default: + fromJSON, err := json.Marshal(fromComponent) + if err != nil { + return result, fmt.Errorf("serializing component at --from:\n%w", err) + } + + toJSON, err := json.Marshal(toComponent) + if err != nil { + return result, fmt.Errorf("serializing component at --to:\n%w", err) + } + + if !bytes.Equal(fromJSON, toJSON) { + result.ChangeType = changeTypeChanged + } + } + + return result, nil +} + +// compareHistoricalSources compares the rendered sources file between two git trees. +func compareHistoricalSources( + fromTree, toTree *object.Tree, + fromRenderedSpecsRelDir, toRenderedSpecsRelDir, name string, +) (bool, error) { + fromRenderedDir, err := components.RenderedSpecDir(fromRenderedSpecsRelDir, name) + if err != nil { + return false, fmt.Errorf("resolving rendered spec dir at --from:\n%w", err) + } + + toRenderedDir, err := components.RenderedSpecDir(toRenderedSpecsRelDir, name) + if err != nil { + return false, fmt.Errorf("resolving rendered spec dir at --to:\n%w", err) + } + + fromSourcesPath := filepath.Join(fromRenderedDir, "sources") + toSourcesPath := filepath.Join(toRenderedDir, "sources") + + fromSources, fromNotFound, fromErr := readFileFromTreeSafe(fromTree, fromSourcesPath) + toSources, toNotFound, toErr := readFileFromTreeSafe(toTree, toSourcesPath) + + if fromErr != nil { + return false, fmt.Errorf("reading sources at --from:\n%w", fromErr) + } + + if toErr != nil { + return false, fmt.Errorf("reading sources at --to:\n%w", toErr) + } + + switch { + case fromNotFound && toNotFound: + return false, nil + case fromNotFound || toNotFound: + return true, nil + default: + return !bytes.Equal(fromSources, toSources), nil + } +} diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go new file mode 100644 index 000000000..aa0135542 --- /dev/null +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go @@ -0,0 +1,306 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "slices" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Tests for 'component changed' in lock-file-free mode, which compares the +// project configuration resolved at each ref instead of stored lock files. + +func TestClassifyHistoricalComponent_BuildFieldChange(t *testing.T) { + fromComponents := map[string]componentComparisonInputs{ + "curl": { + Config: projectconfig.ComponentConfig{ + Build: projectconfig.ComponentBuildConfig{ + Defines: map[string]string{"feature": "disabled"}, + }, + }, + }, + } + toComponents := map[string]componentComparisonInputs{ + "curl": { + Config: projectconfig.ComponentConfig{ + Build: projectconfig.ComponentBuildConfig{ + Defines: map[string]string{"feature": "enabled"}, + }, + }, + }, + } + + result, err := classifyHistoricalComponent("curl", fromComponents, toComponents) + require.NoError(t, err) + assert.Equal(t, changeTypeChanged, result.ChangeType) +} + +func TestClassifyHistoricalComponent_ExcludedFieldsUnchanged(t *testing.T) { + base := projectconfig.ComponentConfig{ + Name: "curl", + SourceConfigFile: &projectconfig.ConfigFile{}, + RenderedSpecDir: "/first/SPECS/c/curl", + Spec: projectconfig.SpecSource{ + Path: "/first/specs/curl.spec", + UpstreamDistro: projectconfig.DistroReference{ + Snapshot: "2025-01-01T00:00:00Z", + }, + }, + Build: projectconfig.ComponentBuildConfig{ + Check: projectconfig.CheckConfig{Skip: true, SkipReason: "first reason"}, + Failure: projectconfig.ComponentBuildFailureConfig{ + Expected: true, + ExpectedReason: "first failure reason", + }, + Hints: projectconfig.ComponentBuildHints{Expensive: true}, + }, + OverlayFiles: []string{"first/*.toml"}, + Publish: projectconfig.ComponentPublishConfig{RPMChannel: "first"}, + Tests: &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Name: "first-test"}}, + }, + Overlays: []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlayAppendSpecLines, + Description: "first description", + Lines: []string{"# functional content"}, + Source: "/first/overlay.patch", + Metadata: &projectconfig.OverlayMetadata{}, + }}, + SourceFiles: []projectconfig.SourceFileReference{{ + Filename: "source.tar.gz", + Hash: "abc123", + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://first.example/source"}, + ReplaceReason: "first replacement reason", + }}, + Packages: map[string]projectconfig.PackageConfig{ + "curl": {Publish: projectconfig.PackagePublishConfig{RPMChannel: "first"}}, + }, + } + updated := base + updated.Name = "renamed metadata" + updated.SourceConfigFile = nil + updated.RenderedSpecDir = "/second/SPECS/c/curl" + updated.Spec.Path = "/second/specs/curl.spec" + updated.Spec.UpstreamDistro.Snapshot = "2026-01-01T00:00:00Z" + updated.Build.Check.SkipReason = "second reason" + updated.Build.Failure.Expected = false + updated.Build.Failure.ExpectedReason = "second failure reason" + updated.Build.Hints.Expensive = false + updated.OverlayFiles = []string{"second/*.toml"} + updated.Publish.RPMChannel = "second" + updated.Tests = &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Name: "second-test"}}, + } + updated.Overlays = slices.Clone(base.Overlays) + updated.Overlays[0].Description = "second description" + updated.Overlays[0].Source = "/second/overlay.patch" + updated.Overlays[0].Metadata = nil + updated.SourceFiles = slices.Clone(base.SourceFiles) + updated.SourceFiles[0].Origin.Type = projectconfig.OriginTypeCustom + updated.SourceFiles[0].Origin.Uri = "https://second.example/source" + updated.SourceFiles[0].ReplaceReason = "second replacement reason" + updated.Packages = map[string]projectconfig.PackageConfig{ + "curl": {Publish: projectconfig.PackagePublishConfig{RPMChannel: "second"}}, + } + + fromComponents := map[string]componentComparisonInputs{ + "curl": {Config: normalizeComponentForComparison(base)}, + } + toComponents := map[string]componentComparisonInputs{ + "curl": {Config: normalizeComponentForComparison(updated)}, + } + + result, err := classifyHistoricalComponent("curl", fromComponents, toComponents) + require.NoError(t, err) + assert.Equal(t, changeTypeUnchanged, result.ChangeType) +} + +func TestBuildComponentComparisonInputs_ContentIdentities(t *testing.T) { + testEnv := testutils.NewTestEnvWithoutLockfile(t) + specPath := "/specs/curl/curl.spec" + overlayPath := "/overlays/fix.patch" + + require.NoError(t, fileutils.WriteFile( + testEnv.FS(), specPath, []byte("Version: 1\n"), fileperms.PublicFile, + )) + require.NoError(t, fileutils.WriteFile( + testEnv.FS(), overlayPath, []byte("first patch\n"), fileperms.PublicFile, + )) + + component := projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: specPath, + }, + Overlays: []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlayAddPatch, + Filename: "fix.patch", + Source: overlayPath, + }}, + } + + first, err := buildComponentComparisonInputs(testEnv.FS(), testEnv.Env, &component) + require.NoError(t, err) + assert.Empty(t, first.Config.Spec.Path) + assert.Contains(t, first.SourceIdentity, "sha256:") + assert.Regexp(t, `^fix\.patch:[0-9a-f]{64}$`, first.OverlaySourceHashes["0"]) + + require.NoError(t, fileutils.WriteFile( + testEnv.FS(), specPath, []byte("Version: 2\n"), fileperms.PublicFile, + )) + second, err := buildComponentComparisonInputs(testEnv.FS(), testEnv.Env, &component) + require.NoError(t, err) + assert.NotEqual(t, first.SourceIdentity, second.SourceIdentity) + assert.Equal(t, first.OverlaySourceHashes, second.OverlaySourceHashes) + + require.NoError(t, fileutils.WriteFile( + testEnv.FS(), overlayPath, []byte("second patch\n"), fileperms.PublicFile, + )) + third, err := buildComponentComparisonInputs(testEnv.FS(), testEnv.Env, &component) + require.NoError(t, err) + assert.Equal(t, second.SourceIdentity, third.SourceIdentity) + assert.NotEqual(t, second.OverlaySourceHashes, third.OverlaySourceHashes) +} + +func TestLoadHistoricalProject_UsesNormalIncludesAndMerging(t *testing.T) { + rootConfig := []byte(`includes = [ + "config/components.toml", + "config/upstream-commits/*.toml", +] + +[project] +default-distro = { name = "testdistro", version = "1.0" } +rendered-specs-dir = "SPECS" + +[distros.testdistro] +description = "Test distro" + +[distros.testdistro.versions."1.0"] +release-ver = "1.0" + +[component-groups.core] +components = ["curl"] +`) + componentConfig := []byte(`[components.curl] +spec = { + type = "upstream", + upstream-distro = { name = "testdistro", version = "1.0" }, + upstream-name = "curl", +} +build = { defines = { feature = "enabled" } } +`) + commitConfig := []byte(`# This file was generated by 'azldev component refresh-upstream-commit' +# Do not edit this file, changes will be lost +# For more details see 'azldev component refresh-upstream-commit --help' +[components.curl.spec] +upstream-commit = "abcdef1234567" +`) + + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{ + "azldev.toml": rootConfig, + "config/components.toml": componentConfig, + "config/upstream-commits/curl.toml": commitConfig, + }}, + }) + + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + + testEnv := testutils.NewTestEnvWithoutLockfile(t) + project, err := loadHistoricalProject(testEnv.Env, tree, ".") + require.NoError(t, err) + + curlConfig, ok := project.components["curl"] + require.True(t, ok) + assert.Equal(t, "abcdef1234567", curlConfig.Spec.UpstreamCommit) + assert.Equal(t, "enabled", curlConfig.Build.Defines["feature"]) + assert.Equal(t, "abcdef1234567", project.comparisonInputs["curl"].SourceIdentity) + assert.Equal(t, "1.0", project.comparisonInputs["curl"].ReleaseVer) + assert.Equal(t, "SPECS", project.renderedSpecsRelDir) + assert.Equal(t, []string{"curl"}, project.componentGroups["core"]) +} + +func TestSelectHistoricalComponentNames_UsesBothRefs(t *testing.T) { + fromProject := &historicalProject{ + components: map[string]projectconfig.ComponentConfig{ + "deleted": {Name: "deleted"}, + "shared": {Name: "shared"}, + }, + componentGroups: map[string][]string{ + "historical": {"deleted"}, + }, + } + toProject := &historicalProject{ + components: map[string]projectconfig.ComponentConfig{ + "added": {Name: "added"}, + "shared": {Name: "shared"}, + }, + componentGroups: map[string][]string{ + "historical": {"added"}, + }, + } + testEnv := testutils.NewTestEnvWithoutLockfile(t) + + allNames, err := selectHistoricalComponentNames( + testEnv.Env, + &components.ComponentFilter{IncludeAllComponents: true}, + fromProject, + toProject, + "/project", + ) + require.NoError(t, err) + assert.Equal(t, []string{"added", "deleted", "shared"}, allNames) + + groupNames, err := selectHistoricalComponentNames( + testEnv.Env, + &components.ComponentFilter{ComponentGroupNames: []string{"historical"}}, + fromProject, + toProject, + "/project", + ) + require.NoError(t, err) + assert.Equal(t, []string{"added", "deleted"}, groupNames) +} + +func TestSelectHistoricalComponentNames_UsesHistoricalSpecPaths(t *testing.T) { + fromProject := &historicalProject{ + components: map[string]projectconfig.ComponentConfig{ + "renamed": { + Name: "renamed", + Spec: projectconfig.SpecSource{Path: "/repo/project/specs/old.spec"}, + }, + }, + } + toProject := &historicalProject{ + components: map[string]projectconfig.ComponentConfig{ + "renamed": { + Name: "renamed", + Spec: projectconfig.SpecSource{Path: "/repo/project/specs/new.spec"}, + }, + }, + } + testEnv := testutils.NewTestEnvWithoutLockfile(t) + + names, err := selectHistoricalComponentNames( + testEnv.Env, + &components.ComponentFilter{SpecPaths: []string{"/project/specs/old.spec"}}, + fromProject, + toProject, + "/", + ) + require.NoError(t, err) + assert.Equal(t, []string{"renamed"}, names) +} + +// --- resolveTree / readFileFromTree / helpers --- diff --git a/internal/app/azldev/cmds/component/cmdoptions.go b/internal/app/azldev/cmds/component/cmdoptions.go new file mode 100644 index 000000000..b347f3ca6 --- /dev/null +++ b/internal/app/azldev/cmds/component/cmdoptions.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/spf13/cobra" +) + +// cmdOptions holds the mode-sensitive choices made when a component command is +// constructed. The zero value describes azldev's default (lock file) mode. +type cmdOptions struct { + // withoutLockfile omits lock-file-specific flags, matching the command surface + // exposed by the global '--without-lockfile' flag. + withoutLockfile bool +} + +// CmdOption customizes how a component command is constructed. +type CmdOption func(*cmdOptions) + +// WithoutLockfileFlags omits the lock-file-specific flags from a component command. +// Commands are registered with this option when the global '--without-lockfile' +// flag selects lock-file-free mode, so the flags that only lock files can honor are +// never offered. +func WithoutLockfileFlags() CmdOption { + return func(options *cmdOptions) { + options.withoutLockfile = true + } +} + +// newCmdOptions resolves the supplied command options. +func newCmdOptions(opts ...CmdOption) cmdOptions { + options := cmdOptions{} + for _, opt := range opts { + opt(&options) + } + + return options +} + +// addComponentFilterOptions registers the component selection flags, including the +// lock-file flags that only apply in azldev's default mode. +func addComponentFilterOptions( + cmd *cobra.Command, filter *components.ComponentFilter, options cmdOptions, +) { + components.AddComponentFilterOptionsToCommand(cmd, filter) + + if !options.withoutLockfile { + components.AddLockValidationFlagToCommand(cmd, filter) + } +} + +// cmdOptionsForApp returns the command options matching the app's selected mode. +func cmdOptionsForApp(app *azldev.App) []CmdOption { + if app.WithoutLockfile() { + return []CmdOption{WithoutLockfileFlags()} + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/component.go b/internal/app/azldev/cmds/component/component.go index 9cfee2ea3..d2694ca63 100644 --- a/internal/app/azldev/cmds/component/component.go +++ b/internal/app/azldev/cmds/component/component.go @@ -27,10 +27,20 @@ components defined in the project configuration.`, buildOnAppInit(app, cmd) changedOnAppInit(app, cmd) diffSourcesOnAppInit(app, cmd) - historyOnAppInit(app, cmd) listOnAppInit(app, cmd) prepareOnAppInit(app, cmd) - queryOnAppInit(app, cmd) renderOnAppInit(app, cmd) - updateOnAppInit(app, cmd) + + // The commands that maintain resolved component state differ by mode: the + // default mode maintains lock files, while lock-file-free mode maintains + // generated upstream-commit config. Registering only the commands that + // belong to the active mode keeps help, docs, and MCP tools honest. + if app.WithoutLockfile() { + legacyOnAppInit(app, cmd) + refreshUpstreamCommitOnAppInit(app, cmd) + } else { + historyOnAppInit(app, cmd) + queryOnAppInit(app, cmd) + updateOnAppInit(app, cmd) + } } diff --git a/internal/app/azldev/cmds/component/component_test.go b/internal/app/azldev/cmds/component/component_test.go index ed2514850..156fe6b36 100644 --- a/internal/app/azldev/cmds/component/component_test.go +++ b/internal/app/azldev/cmds/component/component_test.go @@ -4,6 +4,7 @@ package component_test import ( + "slices" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" @@ -26,3 +27,52 @@ func TestOnAppInit(t *testing.T) { assert.Contains(t, topLevelCommandNames, "component") } + +func TestOnAppInit_CommandsByMode(t *testing.T) { + testCases := []struct { + name string + args []string + refreshExpected bool + }{ + {name: "default mode", args: nil, refreshExpected: false}, + {name: "lock-file-free mode", args: []string{"--without-lockfile"}, refreshExpected: true}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + app := azldev.NewApp(opctx_test.NewMockFileSystemFactory(ctrl), opctx_test.NewMockOSEnvFactory(ctrl)) + + app.PreParseGlobalFlags(testCase.args) + component.OnAppInit(app) + + commandNames, err := app.CommandNames("component") + require.NoError(t, err) + + assert.Equal(t, testCase.refreshExpected, + slices.Contains(commandNames, "refresh-upstream-commit")) + + // The lock-file command names stay reachable in lock-file-free mode, + // where they are registered as hidden no-ops for compatibility. + for _, name := range []string{"history", "query", "update"} { + assert.Contains(t, commandNames, name) + } + }) + } +} + +func TestNewComponentCommands_LockValidationFlagByMode(t *testing.T) { + // The lock-file consistency flags only mean something when lock files are in + // play, so lock-file-free mode leaves them unregistered. + assert.NotNil(t, component.NewBuildCmd().Flags().Lookup("skip-lock-validation")) + assert.NotNil(t, component.NewRenderCmd().Flags().Lookup("skip-lock-validation")) + assert.NotNil(t, component.NewComponentListCommand().Flags().Lookup("skip-lock-validation")) + assert.NotNil(t, component.NewHistoryCmd().Flags().Lookup("skip-lock-validation")) + assert.NotNil(t, component.NewComponentQueryCommand().Flags().Lookup("skip-lock-validation")) + assert.NotNil(t, component.NewUpdateCmd().Flags().Lookup("skip-lock-validation")) + + withoutLockfile := component.WithoutLockfileFlags() + assert.Nil(t, component.NewBuildCmd(withoutLockfile).Flags().Lookup("skip-lock-validation")) + assert.Nil(t, component.NewRenderCmd(withoutLockfile).Flags().Lookup("skip-lock-validation")) + assert.Nil(t, component.NewComponentListCommand(withoutLockfile).Flags().Lookup("skip-lock-validation")) +} diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index 8047bd9fe..56717f5df 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -28,12 +28,12 @@ type DiffSourcesOptions struct { OutputFile string } -func diffSourcesOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewDiffSourcesCmd()) +func diffSourcesOnAppInit(app *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewDiffSourcesCmd(cmdOptionsForApp(app)...)) } // NewDiffSourcesCmd constructs a [cobra.Command] for the "component diff-sources" CLI subcommand. -func NewDiffSourcesCmd() *cobra.Command { +func NewDiffSourcesCmd(opts ...CmdOption) *cobra.Command { var options DiffSourcesOptions cmd := &cobra.Command{ @@ -50,7 +50,7 @@ overlays to the copy and displays the resulting diff between the two trees.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) cmd.Flags().StringVar(&options.OutputFile, "output-file", "", "write the diff output to a file instead of stdout") diff --git a/internal/app/azldev/cmds/component/history.go b/internal/app/azldev/cmds/component/history.go index e5389683f..739d1c2c1 100644 --- a/internal/app/azldev/cmds/component/history.go +++ b/internal/app/azldev/cmds/component/history.go @@ -108,7 +108,7 @@ hand-picking entries to document.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) cmd.Flags().StringVar(&options.SharedTomlMode, "shared", sharedTomlModeShow, "How to report rows for components that share a TOML file with others: "+ diff --git a/internal/app/azldev/cmds/component/history_internal_test.go b/internal/app/azldev/cmds/component/history_internal_test.go index 9e6a339e7..9d0ab12a7 100644 --- a/internal/app/azldev/cmds/component/history_internal_test.go +++ b/internal/app/azldev/cmds/component/history_internal_test.go @@ -157,6 +157,12 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { field := st.Field(i) key := st.Name() + "." + field.Name + // Unexported fields are never fingerprinted: hashstructure skips + // them because it cannot read them by reflection. + if field.PkgPath != "" { + continue + } + // Fields excluded from the fingerprint are operational // metadata (publish channels, build hints, maintenance // markers, etc.), not modifications to upstream. Skip them. diff --git a/internal/app/azldev/cmds/component/legacy.go b/internal/app/azldev/cmds/component/legacy.go new file mode 100644 index 000000000..656d9ba71 --- /dev/null +++ b/internal/app/azldev/cmds/component/legacy.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/spf13/cobra" +) + +const ( + legacyUpdateMessage = "azldev component update no longer does anything and should no longer be used." + legacyHistoryMessage = "azldev component history no longer does anything and should no longer be used." + legacyQueryMessage = "azldev component query no longer does anything and should no longer be used." +) + +func legacyOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand( + newLegacyNoOpCmd("update", nil, legacyUpdateMessage), + newLegacyNoOpCmd("history", []string{"hist"}, legacyHistoryMessage), + newLegacyNoOpCmd("query", nil, legacyQueryMessage), + ) +} + +func newLegacyNoOpCmd(name string, aliases []string, message string) *cobra.Command { + cmd := &cobra.Command{ + Use: name, + Aliases: aliases, + Short: message, + Hidden: true, + DisableFlagParsing: true, + Args: cobra.ArbitraryArgs, + Run: func(cmd *cobra.Command, _ []string) { + cmd.Println(message) + }, + } + + azldev.ExcludeFromMarkdownDocs(cmd) + + return cmd +} diff --git a/internal/app/azldev/cmds/component/legacy_internal_test.go b/internal/app/azldev/cmds/component/legacy_internal_test.go new file mode 100644 index 000000000..50bac09e5 --- /dev/null +++ b/internal/app/azldev/cmds/component/legacy_internal_test.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLegacyCommandsAreHiddenNoOps(t *testing.T) { + tests := []struct { + name string + command string + args []string + expected string + }{ + { + name: "update", + command: "update", + args: []string{"-a", "--bump", "curl"}, + expected: legacyUpdateMessage, + }, + { + name: "history", + command: "history", + args: []string{"-a", "--include-bare", "-O", "json"}, + expected: legacyHistoryMessage, + }, + { + name: "history alias", + command: "hist", + args: []string{"curl"}, + expected: legacyHistoryMessage, + }, + { + name: "query", + command: "query", + args: []string{"-p", "curl", "--arch", "aarch64", "-O", "json"}, + expected: legacyQueryMessage, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + parent := &cobra.Command{Use: "component"} + legacyOnAppInit(nil, parent) + + var output bytes.Buffer + parent.SetOut(&output) + parent.SetErr(&output) + parent.SetArgs(append([]string{test.command}, test.args...)) + + require.NoError(t, parent.Execute()) + assert.Equal(t, test.expected+"\n", output.String()) + + command, _, err := parent.Find([]string{test.command}) + require.NoError(t, err) + assert.True(t, command.Hidden) + assert.True(t, command.DisableFlagParsing) + }) + } +} diff --git a/internal/app/azldev/cmds/component/list.go b/internal/app/azldev/cmds/component/list.go index 04cc0e7f2..4f0ba3ee1 100644 --- a/internal/app/azldev/cmds/component/list.go +++ b/internal/app/azldev/cmds/component/list.go @@ -19,12 +19,12 @@ type ListComponentOptions struct { ComponentFilter components.ComponentFilter } -func listOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewComponentListCommand()) +func listOnAppInit(app *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewComponentListCommand(cmdOptionsForApp(app)...)) } // Constructs a [cobra.Command] for "component list" CLI subcommand. -func NewComponentListCommand() *cobra.Command { +func NewComponentListCommand(opts ...CmdOption) *cobra.Command { options := &ListComponentOptions{} cmd := &cobra.Command{ @@ -55,11 +55,14 @@ Component name patterns support glob syntax (*, ?, []).`, azldev.ExportAsReadOnlyMCPTool(cmd) - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + cmdOptions := newCmdOptions(opts...) + addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions) // List always skips lock validation (read-only), so the flag is // meaningless here. Hide it to avoid confusion. - _ = cmd.Flags().MarkHidden("skip-lock-validation") + if !cmdOptions.withoutLockfile { + _ = cmd.Flags().MarkHidden("skip-lock-validation") + } return cmd } diff --git a/internal/app/azldev/cmds/component/orphans.go b/internal/app/azldev/cmds/component/orphans.go new file mode 100644 index 000000000..fadc24e92 --- /dev/null +++ b/internal/app/azldev/cmds/component/orphans.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "fmt" + "log/slog" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" +) + +// orphanStore is the per-component state store that orphan reconciliation acts on. +// Both the lock store and the generated upstream-commit store satisfy it through +// small adapters, so the reconciliation policy lives in one place regardless of the +// mode's storage. +type orphanStore interface { + // findOrphans returns the stored entries with no matching resolved component. + findOrphans(resolved map[string]projectconfig.ComponentConfig) ([]string, error) + + // pruneOrphans deletes those entries and returns how many were removed. + pruneOrphans(resolved map[string]projectconfig.ComponentConfig) (int, error) +} + +// orphanMessages holds the mode-specific wording used while reconciling orphans. +type orphanMessages struct { + // noComponents describes the state that would be treated as orphaned when the + // resolved component set is empty. It is completed with "would be" or "will be" + // depending on whether the run is check-only. + noComponents string + + // pruned describes what was removed, for the summary log line. + pruned string +} + +// handleOrphans reconciles a store of per-component state with the resolved +// component set. In normal mode it deletes orphan entries; in check-only mode it +// returns the entries that would be deleted without touching disk. Returns +// (nil, nil) when not running with --all-components, since orphan handling is +// scoped to whole-set runs. +func handleOrphans( + store orphanStore, + comps []components.Component, + includeAllComponents bool, + checkOnly bool, + messages orphanMessages, +) ([]string, error) { + if !includeAllComponents { + return nil, nil + } + + if len(comps) == 0 { + tense := "will be" + if checkOnly { + tense = "would be" + } + + slog.Warn(fmt.Sprintf("No components resolved; %s %s treated as orphans", + messages.noComponents, tense)) + } + + resolvedNames := make(map[string]projectconfig.ComponentConfig, len(comps)) + for _, comp := range comps { + resolvedNames[comp.GetName()] = *comp.GetConfig() + } + + if checkOnly { + return store.findOrphans(resolvedNames) + } + + pruned, pruneErr := store.pruneOrphans(resolvedNames) + if pruneErr != nil { + return nil, pruneErr + } + + if pruned > 0 { + slog.Info(messages.pruned, "count", pruned) + } + + return nil, nil +} diff --git a/internal/app/azldev/cmds/component/preparerctx.go b/internal/app/azldev/cmds/component/preparerctx.go new file mode 100644 index 000000000..f98f7b671 --- /dev/null +++ b/internal/app/azldev/cmds/component/preparerctx.go @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" +) + +// gitRepoPreparerOptions returns the [sources.PreparerOption] values that enable +// synthetic dist-git history for the current mode. +// +// By default, history is derived from the component's lock file, and working-tree +// changes are detected by comparing input fingerprints. In lock-file-free mode it +// is derived from the component's generated upstream-commit TOML, which has no +// fingerprint to compare against, so only committed changes are represented. +func gitRepoPreparerOptions( + env *azldev.Env, distro sourceproviders.ResolvedDistro, +) []sources.PreparerOption { + if env.WithoutLockfile() { + return []sources.PreparerOption{ + sources.WithGitRepo(env, nil, ""), + sources.WithoutLockfileHistory(), + } + } + + return []sources.PreparerOption{ + sources.WithGitRepo(env, env.LockReader(), distro.Version.ReleaseVer), + sources.WithDirtyDetection(), + } +} diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index ac48bdcac..3e17f34e8 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -27,11 +27,11 @@ type PrepareSourcesOptions struct { SkipSources bool } -func prepareOnAppInit(_ *azldev.App, sourceCmd *cobra.Command) { - sourceCmd.AddCommand(NewPrepareSourcesCmd()) +func prepareOnAppInit(app *azldev.App, sourceCmd *cobra.Command) { + sourceCmd.AddCommand(NewPrepareSourcesCmd(cmdOptionsForApp(app)...)) } -func NewPrepareSourcesCmd() *cobra.Command { +func NewPrepareSourcesCmd(opts ...CmdOption) *cobra.Command { var options PrepareSourcesOptions cmd := &cobra.Command{ @@ -62,7 +62,7 @@ Only one component may be selected at a time.`, }, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", "output directory") _ = cmd.MarkFlagRequired("output-dir") @@ -155,10 +155,7 @@ func buildPreparerOptions( var opts []sources.PreparerOption if !options.WithoutGitRepo && !options.SkipOverlays { - opts = append(opts, - sources.WithGitRepo(env, env.LockReader(), distro.Version.ReleaseVer), - sources.WithDirtyDetection(), - ) + opts = append(opts, gitRepoPreparerOptions(env, distro)...) } opts = append(opts, diff --git a/internal/app/azldev/cmds/component/query.go b/internal/app/azldev/cmds/component/query.go index a4bdaa353..83014b12c 100644 --- a/internal/app/azldev/cmds/component/query.go +++ b/internal/app/azldev/cmds/component/query.go @@ -69,7 +69,7 @@ The rendered-specs-dir must exist on disk; if it doesn't, run ValidArgsFunction: components.GenerateComponentNameCompletions, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) cmd.Flags().Var(&options.Arch, "arch", "Target architecture passed to rpmspec via --target (x86_64, aarch64). "+ diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit.go b/internal/app/azldev/cmds/component/refreshupstreamcommit.go new file mode 100644 index 000000000..a3fd1bdeb --- /dev/null +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit.go @@ -0,0 +1,626 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "fmt" + "log/slog" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/upstreamcommit" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/parmap" + "github.com/spf13/cobra" +) + +// RefreshUpstreamCommitOptions holds options for the component refresh-upstream-commit command. +type RefreshUpstreamCommitOptions struct { + ComponentFilter components.ComponentFilter + // UpstreamCommitsDir is the project-relative or absolute directory containing + // generated per-component TOML files. + UpstreamCommitsDir string + // CheckOnly resolves upstream commits but does not write TOML files or + // prune orphans. + CheckOnly bool +} + +const defaultUpstreamCommitsDir = upstreamcommit.DefaultDir + +func refreshUpstreamCommitOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewRefreshUpstreamCommitCmd()) +} + +// NewRefreshUpstreamCommitCmd constructs the "component refresh-upstream-commit" command. +func NewRefreshUpstreamCommitCmd() *cobra.Command { + options := &RefreshUpstreamCommitOptions{} + options.UpstreamCommitsDir = defaultUpstreamCommitsDir + + cmd := &cobra.Command{ + Use: "refresh-upstream-commit", + Short: "Resolve and record upstream commits for components", + Long: `Resolve upstream commits for components and write normal per-component TOML configuration. + +For upstream components, this resolves the effective commit hash using the +distro snapshot time, then records it as spec.upstream-commit in +base/upstream-commits/.toml by default. Include that directory's TOML +files before component-specific TOML configuration so subsequent commands use +the generated commit unless the component configuration explicitly overrides +it. + +All TOML configuration is resolved before the source type is checked. Selected +components whose effective spec.type is not "upstream" do not contact an +upstream provider, and any generated upstream-commit TOML for them is removed. +Configuration is loaded permissively for this command so stale generated pins +cannot prevent cleanup after a component is removed or changed to another +source type. Other configuration validation failures are reported as warnings. + +When updating all components (-a), orphan generated TOML files are +automatically pruned. +Orphan pruning is skipped when updating individual components to avoid +accidentally removing files for components not included in the filter. + +The --check-only flag runs the full pipeline but does NOT write TOML files or +prune orphans. The command exits 0 when nothing would change and exits 1 when +any component is stale or any generated TOML would be pruned. Intended for CI gates.`, + Example: ` # Refresh all components + azldev --without-lockfile component refresh-upstream-commit -a + + # Refresh a single component + azldev --without-lockfile component refresh-upstream-commit -p curl + + # Refresh components in a group + azldev --without-lockfile component refresh-upstream-commit -g core + + # Write generated files to a custom directory + azldev --without-lockfile component refresh-upstream-commit -a --upstream-commits-dir config/commits + + # CI gate: exit 0 if commit TOMLs are current, 1 if anything would change + azldev --without-lockfile component refresh-upstream-commit -a --check-only -q`, + RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { + options.ComponentFilter.ComponentNamePatterns = append( + args, options.ComponentFilter.ComponentNamePatterns..., + ) + + return RefreshUpstreamCommits(env, options) + }), + ValidArgsFunction: components.GenerateComponentNameCompletions, + Annotations: map[string]string{ + azldev.CommandAnnotationPermissiveConfig: "true", + }, + } + + components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + + cmd.Flags().StringVar(&options.UpstreamCommitsDir, "upstream-commits-dir", + defaultUpstreamCommitsDir, + "directory for generated per-component upstream-commit TOML files") + _ = cmd.MarkFlagDirname("upstream-commits-dir") + cmd.Flags().BoolVar(&options.CheckOnly, "check-only", false, + "resolve upstream commits but do not write TOML files or prune orphans. "+ + "Exits 0 when nothing would change and 1 when any component is stale "+ + "(or, with --all-components, when any orphan generated TOML "+ + "would be pruned). Intended for CI gates") + + return cmd +} + +// RefreshUpstreamCommitResult is the per-component output for the refresh command. +type RefreshUpstreamCommitResult struct { + Component string `json:"component" table:",sortkey"` + UpstreamCommit string `json:"upstreamCommit,omitempty"` + PreviousCommit string `json:"previousCommit,omitempty" table:"-"` + Changed bool `json:"changed"` + Removed bool `json:"removed,omitempty" table:",omitempty"` + Skipped bool `json:"skipped,omitempty"` + SkipReason string `json:"skipReason,omitempty" table:",omitempty"` + Error string `json:"error,omitempty" table:",omitempty"` +} + +// RefreshUpstreamCommits resolves upstream commits for all selected components and +// writes the results to per-component TOML files. +func RefreshUpstreamCommits( + env *azldev.Env, options *RefreshUpstreamCommitOptions, +) ([]RefreshUpstreamCommitResult, error) { + resolver := components.NewResolver(env) + + resolved, err := resolver.FindComponents(&options.ComponentFilter) + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + allComps := resolved.Components() + if len(allComps) == 0 && !options.ComponentFilter.IncludeAllComponents { + return nil, errors.New("no components matched the filter") + } + + if env.ProjectDir() == "" { + return nil, errors.New("no project directory configured; cannot refresh upstream commit TOML files") + } + + commitDir := options.UpstreamCommitsDir + if commitDir == "" { + commitDir = defaultUpstreamCommitsDir + } + + if !filepath.IsAbs(commitDir) { + commitDir = filepath.Join(env.ProjectDir(), commitDir) + } + + store := upstreamcommit.NewStore(env.FS(), commitDir) + + // Configuration is fully parsed and merged before deciding whether each + // selected component has an upstream identity. Non-upstream components + // never contact a provider; an existing generated pin is marked for + // removal instead. + upstreamComps, results, inspectErr := inspectSelectedComponents(allComps, store) + if inspectErr != nil { + return results, inspectErr + } + + results = append(results, resolveUpstreamCommitsParallel(env, upstreamComps, store)...) + + // Don't save if the context was cancelled (Ctrl+C). + if env.Context().Err() != nil { + return results, errors.New("refresh cancelled; upstream commit TOML files not updated") + } + + // Check results and bail on errors before saving. + if err := checkRefreshErrors(results); err != nil { + return filterRefreshDisplayResults(results), err + } + + // Write per-component TOML files only on full success. + if err := saveUpstreamCommitConfigs(store, results, options.CheckOnly); err != nil { + return results, err + } + + // Skipped in --check-only mode -- the "changed" counter would lie about a + // run that wrote nothing, and the structured error returned below already + // names every affected component. + if !options.CheckOnly { + logRefreshSummary(results) + } + + // Prune orphan generated TOML files when updating all components. + // Use the resolved component set (not raw config) to include + // spec-glob-discovered components that aren't in config directly. + // Generated TOMLs are version controlled, so pruning is safe even if the + // resolved set is empty (e.g., all components removed from config). + wouldPrune, orphanErr := handleOrphanConfigs(store, allComps, options) + if orphanErr != nil { + return filterRefreshDisplayResults(results), orphanErr + } + + if options.CheckOnly { + wouldPrune = excludePendingRemovals(wouldPrune, results) + + return refreshCheckOnlyResult(results, wouldPrune) + } + + // Filter results for table output: show changed and skipped components. + return filterRefreshDisplayResults(results), nil +} + +func inspectSelectedComponents( + comps []components.Component, + store *upstreamcommit.Store, +) ([]components.Component, []RefreshUpstreamCommitResult, error) { + upstream := make([]components.Component, 0, len(comps)) + + var results []RefreshUpstreamCommitResult + + for _, comp := range comps { + if comp.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeUpstream { + upstream = append(upstream, comp) + + continue + } + + exists, err := store.Exists(comp.GetName()) + if err != nil { + return nil, results, fmt.Errorf( + "checking generated upstream commit TOML for non-upstream component %#q:\n%w", + comp.GetName(), err, + ) + } + + if exists { + results = append(results, RefreshUpstreamCommitResult{ + Component: comp.GetName(), + Changed: true, + Removed: true, + }) + } + } + + return upstream, results, nil +} + +func excludePendingRemovals( + orphans []string, results []RefreshUpstreamCommitResult, +) []string { + removed := make(map[string]struct{}) + + for idx := range results { + if results[idx].Removed { + removed[results[idx].Component] = struct{}{} + } + } + + filtered := make([]string, 0, len(orphans)) + for _, orphan := range orphans { + if _, found := removed[orphan]; !found { + filtered = append(filtered, orphan) + } + } + + return filtered +} + +// handleOrphanConfigs reconciles the generated TOML directory with the resolved +// component set. In normal mode it deletes orphan files; in --check-only +// mode it returns the list of orphans that would be deleted without +// touching disk. Returns (nil, nil) when not running with --all-components, +// since orphan handling is scoped to whole-set updates. +func handleOrphanConfigs( + store *upstreamcommit.Store, + comps []components.Component, + options *RefreshUpstreamCommitOptions, +) ([]string, error) { + return handleOrphans( + upstreamCommitOrphanStore{store: store}, + comps, + options.ComponentFilter.IncludeAllComponents, + options.CheckOnly, + orphanMessages{ + noComponents: "all generated upstream commit TOMLs", + pruned: "Pruned orphan upstream commit TOMLs", + }, + ) +} + +// upstreamCommitOrphanStore adapts [upstreamcommit.Store] to [orphanStore]. +type upstreamCommitOrphanStore struct { + store *upstreamcommit.Store +} + +func (s upstreamCommitOrphanStore) findOrphans( + resolved map[string]projectconfig.ComponentConfig, +) ([]string, error) { + orphans, err := s.store.FindOrphans(resolved) + if err != nil { + return nil, fmt.Errorf("finding orphan upstream commit TOMLs:\n%w", err) + } + + return orphans, nil +} + +func (s upstreamCommitOrphanStore) pruneOrphans( + resolved map[string]projectconfig.ComponentConfig, +) (int, error) { + pruned, err := s.store.PruneOrphans(resolved) + if err != nil { + return 0, fmt.Errorf("pruning orphan upstream commit TOMLs:\n%w", err) + } + + return pruned, nil +} + +// refreshCheckOnlyResult inspects the results of a '--check-only' refresh run and +// returns (results, error) when any component would change or any generated TOML +// would be pruned. The error names the affected components so CI logs are +// useful at a glance. Returns (results, nil) when nothing would change -- +// the caller exits 0. Results are returned in both cases so structured +// consumers (e.g. -O json) retain the per-component data the pipeline just +// computed. +func refreshCheckOnlyResult( + results []RefreshUpstreamCommitResult, wouldPrune []string, +) ([]RefreshUpstreamCommitResult, error) { + var changed []string + + for idx := range results { + if results[idx].Changed { + changed = append(changed, results[idx].Component) + } + } + + display := filterRefreshDisplayResults(results) + + if len(changed) == 0 && len(wouldPrune) == 0 { + return display, nil + } + + var parts []string + if len(changed) > 0 { + parts = append(parts, fmt.Sprintf("%d component(s) would change: %s", + len(changed), strings.Join(changed, ", "))) + } + + if len(wouldPrune) > 0 { + parts = append(parts, fmt.Sprintf("%d orphan upstream commit TOML file(s) would be pruned: %s", + len(wouldPrune), strings.Join(wouldPrune, ", "))) + } + + return display, fmt.Errorf("upstream commit TOML files are stale; %s. "+ + "Run 'azldev --without-lockfile component refresh-upstream-commit -a' to refresh", + strings.Join(parts, "; ")) +} + +// saveUpstreamCommitConfigs writes TOML files for changed upstream commits. +func saveUpstreamCommitConfigs( + store *upstreamcommit.Store, results []RefreshUpstreamCommitResult, checkOnly bool, +) error { + saved := make([]string, 0, len(results)) + + // Log partially-saved components on any error so the user knows which + // TOML files were written before the failure. + var retErr error + + defer func() { + if retErr != nil && len(saved) > 0 { + slog.Info("Upstream commit TOMLs saved before failure", "components", saved) + } + }() + + for idx := range results { + if results[idx].Error != "" || results[idx].Skipped { + continue + } + + written, err := applyRefreshResult(store, &results[idx], checkOnly) + if err != nil { + retErr = err + + return retErr + } + + if written { + saved = append(saved, results[idx].Component) + } + } + + return nil +} + +// applyRefreshResult writes one changed TOML file. The returned 'written' +// flag is always false in check-only mode. +func applyRefreshResult( + store *upstreamcommit.Store, result *RefreshUpstreamCommitResult, checkOnly bool, +) (bool, error) { + if !result.Changed { + return false, nil + } + + // In check-only mode the caller wants to know what *would* change without + // touching disk. Skip the write but keep result.Changed flipped so the + // caller can build the user-visible diff list. + if checkOnly { + return false, nil + } + + if result.Removed { + removed, removeErr := store.Remove(result.Component) + if removeErr != nil { + return false, fmt.Errorf( + "removing upstream commit TOML for non-upstream component %#q:\n%w", + result.Component, removeErr, + ) + } + + return removed, nil + } + + if saveErr := store.Save(result.Component, result.UpstreamCommit); saveErr != nil { + return false, fmt.Errorf("saving upstream commit TOML for %#q:\n%w", result.Component, saveErr) + } + + return true, nil +} + +// checkRefreshErrors returns an error if any component failed to resolve. +// Does NOT log a summary; call [logRefreshSummary] after saves are complete. +func checkRefreshErrors(results []RefreshUpstreamCommitResult) error { + var failedNames []string + + for idx := range results { + if results[idx].Error != "" { + failedNames = append(failedNames, results[idx].Component) + } + } + + if len(failedNames) > 0 { + slog.Error("Refresh failed", + "total", len(results), + "errors", len(failedNames)) + + return fmt.Errorf( + "%d component(s) failed to resolve; upstream commit TOML files not updated:\n %s", + len(failedNames), strings.Join(failedNames, "\n ")) + } + + return nil +} + +// logRefreshSummary logs the final refresh summary. +func logRefreshSummary(results []RefreshUpstreamCommitResult) { + var changed, skipped, upToDate int + + for idx := range results { + switch { + case results[idx].Skipped: + skipped++ + case results[idx].Changed: + changed++ + default: + upToDate++ + } + } + + slog.Info("Refresh complete", + "total", len(results), + "changed", changed, + "upToDate", upToDate, + "skipped", skipped) +} + +// filterRefreshDisplayResults returns changed, skipped, and errored results for table +// display. Up-to-date components (not Changed, not Skipped, no Error) are +// excluded — they represent the common "nothing to do" case and would dominate +// the output. Errored entries are kept so the user can see what failed when +// the command exits non-zero via the partial-results-on-error path. +func filterRefreshDisplayResults( + results []RefreshUpstreamCommitResult, +) []RefreshUpstreamCommitResult { + var tableResults []RefreshUpstreamCommitResult + + for idx := range results { + if results[idx].Changed || results[idx].Skipped || results[idx].Error != "" { + tableResults = append(tableResults, results[idx]) + } + } + + return tableResults +} + +func resolveUpstreamCommitsParallel( + env *azldev.Env, + comps []components.Component, + store *upstreamcommit.Store, +) []RefreshUpstreamCommitResult { + results := make([]RefreshUpstreamCommitResult, len(comps)) + + progressEvent := env.StartEvent("Resolving upstream commits", "count", len(comps)) + defer progressEvent.End() + + workerEnv, cancel := env.WithCancel() + defer cancel() + + // Resolve every selected upstream component instead of inferring freshness + // from duplicated metadata. The provider is the authority for the commit + // selected by the snapshot, and that result is compared directly with the + // generated TOML before deciding whether a write is needed. + parallel := make([]refreshParallelItem, len(comps)) + for idx, comp := range comps { + results[idx].Component = comp.GetName() + parallel[idx] = refreshParallelItem{idx: idx, comp: comp} + } + + // Each resolution may involve network I/O, so we parallelize. + parmapResults := parmap.Map( + workerEnv, + env.FastConcurrency(), + parallel, + func(done, _ int) { + progressEvent.SetProgress(int64(done), int64(len(comps))) + }, + func(ctx context.Context, item refreshParallelItem) struct{} { + resolveAndRecordCommit(ctx, workerEnv, cancel, item.comp, store, &results[item.idx]) + + return struct{}{} + }, + ) + + // Items that never acquired a worker slot (ctx cancelled mid-flight) get + // marked Skipped — matches the legacy semaphore-select behaviour. + for i, pr := range parmapResults { + if pr.Cancelled { + idx := parallel[i].idx + results[idx].Skipped = true + results[idx].SkipReason = "cancelled" + } + } + + return results +} + +// refreshParallelItem pairs a component with its result index for parmap workers. +type refreshParallelItem struct { + idx int + comp components.Component +} + +// resolveAndRecordCommit resolves one component's upstream commit. +func resolveAndRecordCommit( + ctx context.Context, + env *azldev.Env, + cancel context.CancelFunc, + comp components.Component, + store *upstreamcommit.Store, + result *RefreshUpstreamCommitResult, +) { + // Clear the loaded pin before asking the provider to resolve. Render and + // build honor Spec.UpstreamCommit for reproducibility, but refresh is the + // operation that advances that pin: leaving the old value in place would + // make the provider return it immediately and a newer snapshot could never + // move the component forward. + comp.GetConfig().Spec.UpstreamCommit = "" + + commit, resolveErr := resolveUpstreamCommit(ctx, env, comp) + if resolveErr != nil { + result.Error = resolveErr.Error() + + // Cancel remaining goroutines on first real failure. + cancel() + + return + } + + result.UpstreamCommit = commit + + checkConfigChanged(store, comp.GetName(), result) +} + +// checkConfigChanged compares the resolved commit with the generated TOML. +func checkConfigChanged( + store *upstreamcommit.Store, componentName string, result *RefreshUpstreamCommitResult, +) { + existingCommit, exists, loadErr := store.Get(componentName) + if loadErr != nil { + result.Error = fmt.Sprintf("loading upstream commit TOML: %v", loadErr) + + return + } + + if !exists { + result.Changed = true + + return + } + + result.PreviousCommit = existingCommit + result.Changed = existingCommit != result.UpstreamCommit +} + +func resolveUpstreamCommit( + ctx context.Context, + env *azldev.Env, + comp components.Component, +) (string, error) { + componentName := comp.GetName() + + distro, err := sourceproviders.ResolveDistro(env, comp) + if err != nil { + return "", fmt.Errorf("resolving distro for %#q:\n%w", componentName, err) + } + + sourceManager, err := sourceproviders.NewSourceManager(env, distro) + if err != nil { + return "", fmt.Errorf("creating source manager for %#q:\n%w", componentName, err) + } + + commit, err := sourceManager.ResolveSourceIdentity(ctx, comp) + if err != nil { + return "", fmt.Errorf("resolving upstream commit for %#q:\n%w", componentName, err) + } + + slog.Debug("Resolved upstream commit", "component", componentName, "commit", commit) + + return commit, nil +} diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_internal_test.go new file mode 100644 index 000000000..87c2653e4 --- /dev/null +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_internal_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/upstreamcommit" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testUpstreamCommitsDir = "/project/base/upstream-commits" + +func TestSaveUpstreamCommitConfigs_WritesChangedCommit(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + results := []RefreshUpstreamCommitResult{{ + Component: "curl", + UpstreamCommit: "abc123", + Changed: true, + }} + + require.NoError(t, saveUpstreamCommitConfigs(store, results, false)) + + commit, exists, err := store.Get("curl") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "abc123", commit) +} + +func TestSaveUpstreamCommitConfigs_SkipsUnchangedAndFailed(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + results := []RefreshUpstreamCommitResult{ + {Component: "unchanged"}, + {Component: "errored", Changed: true, Error: "resolution failed"}, + {Component: "skipped", Changed: true, Skipped: true, SkipReason: "cancelled"}, + } + + require.NoError(t, saveUpstreamCommitConfigs(store, results, false)) + + for _, componentName := range []string{"unchanged", "errored", "skipped"} { + _, exists, err := store.Get(componentName) + require.NoError(t, err) + assert.False(t, exists) + } +} diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_resolution_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_resolution_test.go new file mode 100644 index 000000000..8a27bdad1 --- /dev/null +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_resolution_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component_test + +import ( + "testing" + + componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRefreshUpstreamCommitAlwaysResolvesUpstreamComponent(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + gitCalls := setupMockGitWithCounter(env, "aabbccdd11223344") + addRefreshUpstreamComponent(env, "curl") + + options := &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + } + + _, err := componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err) + require.Positive(t, gitCalls.Load()) + + gitCalls.Store(0) + + results, err := componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err) + assert.Positive(t, gitCalls.Load(), "repeated updates must re-resolve upstream state") + + for _, result := range results { + if result.Component == "curl" { + assert.False(t, result.Changed) + } + } +} diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go new file mode 100644 index 000000000..c3e2de99b --- /dev/null +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go @@ -0,0 +1,502 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/upstreamcommit" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testUpstreamCommitsDir = "/project/base/upstream-commits" + +func TestNewRefreshUpstreamCommitCmd(t *testing.T) { + cmd := componentcmds.NewRefreshUpstreamCommitCmd() + require.NotNil(t, cmd) + assert.Equal(t, "refresh-upstream-commit", cmd.Use) + assert.NotNil(t, cmd.RunE) + assert.Nil(t, cmd.Flags().Lookup("bump")) + assert.Contains(t, cmd.Annotations, azldev.CommandAnnotationPermissiveConfig) +} + +func TestNewRefreshUpstreamCommitCmd_Flags(t *testing.T) { + cmd := componentcmds.NewRefreshUpstreamCommitCmd() + + allFlag := cmd.Flags().Lookup("all-components") + require.NotNil(t, allFlag, "all-components flag should be registered") + + componentFlag := cmd.Flags().Lookup("component") + require.NotNil(t, componentFlag, "component flag should be registered") +} + +func TestRefreshUpstreamCommitCmd_NoComponents(t *testing.T) { + testEnv := testutils.NewTestEnvWithoutLockfile(t) + + cmd := componentcmds.NewRefreshUpstreamCommitCmd() + cmd.SetArgs([]string{"nonexistent-component"}) + + err := cmd.ExecuteContext(testEnv.Env) + + require.Error(t, err) + assert.Contains(t, err.Error(), "component not found") +} + +func TestRefreshUpstreamCommitCmd_CleansPinsThatInvalidateStrictConfig(t *testing.T) { + testCases := []struct { + name string + componentConfig string + }{ + { + name: "removed component", + }, + { + name: "converted to local", + componentConfig: `[components.test-component.spec] +type = "local" +path = "../../specs/test-component.spec" +`, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + testEnv := testutils.NewTestEnvWithoutLockfile(t) + + rootConfig := `includes = [ + "base/upstream-commits/*.toml", + "base/components/*.toml", +] + +[project] +default-distro = { name = "test-distro", version = "1.0" } + +[distros.test-distro] +default-version = "1.0" + +[distros.test-distro.versions."1.0"] +release-ver = "1.0" +` + require.NoError(t, fileutils.WriteFile( + testEnv.TestFS, + "/project/azldev.toml", + []byte(rootConfig), + fileperms.PublicFile, + )) + + if testCase.componentConfig != "" { + require.NoError(t, fileutils.WriteFile( + testEnv.TestFS, + "/project/base/components/test-component.toml", + []byte(testCase.componentConfig), + fileperms.PublicFile, + )) + } + + store := upstreamcommit.NewStore(testEnv.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("test-component", "abc1234")) + + app := azldev.NewApp( + testEnv.TestInterfaces.FileSystemFactory, + testEnv.TestInterfaces.OSEnvFactory, + ) + + // The command is only registered in lock-file-free mode, which the + // CLI selects by pre-parsing the global flags before registration. + args := []string{ + "--without-lockfile", "component", "refresh-upstream-commit", "--all-components", + } + + app.PreParseGlobalFlags(args) + componentcmds.OnAppInit(app) + + exitCode := app.Execute(args) + require.Zero(t, exitCode) + + _, exists, err := store.Get("test-component") + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +// addRefreshUpstreamComponent adds an upstream component to the test config +// without pre-populating generated commit TOML. +func addRefreshUpstreamComponent(env *testutils.TestEnv, name string) { + env.Config.Components[name] = projectconfig.ComponentConfig{ + Name: name, + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeUpstream, + }, + } +} + +// TestRefreshUpstreamCommits_WritesCommit exercises the full refresh pipeline. +func TestRefreshUpstreamCommits_WritesCommit(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + const commit = "abc123def456" + + setupMockGit(env, commit) + addRefreshUpstreamComponent(env, "curl") + + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.True(t, results[0].Changed) + assert.Equal(t, commit, results[0].UpstreamCommit) + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + + savedCommit, exists, loadErr := store.Get("curl") + require.NoError(t, loadErr) + assert.True(t, exists) + assert.Equal(t, commit, savedCommit) +} + +func TestRefreshUpstreamCommits_ConfigOnlyChangeDoesNotChangeCommitTOML(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + const commit = "abc123def456" + + setupMockGit(env, commit) + addRefreshUpstreamComponent(env, "curl") + + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + + options := &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + } + + results, err := componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err) + require.Len(t, results, 1) + assert.True(t, results[0].Changed) + + modifiedConfig := env.Config.Components["curl"] + modifiedConfig.Build.With = []string{"ssl"} + env.Config.Components["curl"] = modifiedConfig + + results, err = componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err) + assert.Empty(t, results) + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + savedCommit, exists, err := store.Get("curl") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, commit, savedCommit) +} + +// TestRefreshUpstreamCommits_MultipleComponents tests refreshing multiple components. +func TestRefreshUpstreamCommits_MultipleComponents(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + const commit = "multi-commit-hash" + + setupMockGit(env, commit) + addRefreshUpstreamComponent(env, "curl") + addRefreshUpstreamComponent(env, "bash") + + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + + // Should have results for both (may include skipped too). + var changedNames []string + + for _, r := range results { + if r.Changed { + changedNames = append(changedNames, r.Component) + } + } + + assert.Contains(t, changedNames, "curl") + assert.Contains(t, changedNames, "bash") + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + + curlCommit, curlExists, err := store.Get("curl") + require.NoError(t, err) + bashCommit, bashExists, err := store.Get("bash") + require.NoError(t, err) + assert.True(t, curlExists) + assert.True(t, bashExists) + assert.Equal(t, commit, curlCommit) + assert.Equal(t, commit, bashCommit) +} + +func TestRefreshUpstreamCommits_LocalComponentDoesNotWriteCommitTOML(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + env.Config.Components["local-pkg"] = projectconfig.ComponentConfig{ + Name: "local-pkg", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: "/project/specs/local-pkg/local-pkg.spec", + }, + } + + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + assert.Empty(t, results) + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + _, exists, err := store.Get("local-pkg") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestRefreshUpstreamCommits_NonUpstreamComponentRemovesGeneratedCommitTOML(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + env.Config.Components["local-pkg"] = projectconfig.ComponentConfig{ + Name: "local-pkg", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: "/project/specs/local-pkg/local-pkg.spec", + }, + } + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("local-pkg", "stale-commit")) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{ + ComponentNamePatterns: []string{"local-pkg"}, + }, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "local-pkg", results[0].Component) + assert.True(t, results[0].Changed) + assert.True(t, results[0].Removed) + + _, exists, err := store.Get("local-pkg") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestRefreshUpstreamCommits_CheckOnlyDetectsNonUpstreamGeneratedCommitTOML(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + env.Config.Components["local-pkg"] = projectconfig.ComponentConfig{ + Name: "local-pkg", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: "/project/specs/local-pkg/local-pkg.spec", + }, + } + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("local-pkg", "stale-commit")) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{ + ComponentNamePatterns: []string{"local-pkg"}, + }, + CheckOnly: true, + }) + require.ErrorContains(t, err, "local-pkg") + require.Len(t, results, 1) + assert.True(t, results[0].Changed) + assert.True(t, results[0].Removed) + + _, exists, err := store.Get("local-pkg") + require.NoError(t, err) + assert.True(t, exists, "--check-only must not remove generated TOML files") +} + +// TestRefreshUpstreamCommits_AdvancesStaleCommit is a regression test for the case +// where a generated pin is at commit A and the snapshot resolves to +// commit B must result in B being written (not A echoed back). Without +// clearing the configured pin before re-resolution, source resolution would +// return A and the generated TOML would never advance. +func TestRefreshUpstreamCommits_AdvancesStaleCommit(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + const initialCommit = "initial-aaa111" + + const advancedCommit = "advanced-bbb222" + + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("curl", initialCommit)) + + addRefreshUpstreamComponent(env, "curl") + + // Mock git now resolves to a NEW commit — upstream moved. + setupMockGit(env, advancedCommit) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + require.Len(t, results, 1) + + assert.Equal(t, advancedCommit, results[0].UpstreamCommit, + "refresh must re-resolve and return the advanced commit, not echo the configured one") + assert.True(t, results[0].Changed, "generated commit advanced") + assert.Equal(t, initialCommit, results[0].PreviousCommit, + "PreviousCommit should track the prior generated TOML") + + freshStore := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + updatedCommit, exists, loadErr := freshStore.Get("curl") + require.NoError(t, loadErr) + assert.True(t, exists) + assert.Equal(t, advancedCommit, updatedCommit) +} + +// TestRefreshUpstreamCommits_CheckOnly_StaleReturnsError verifies that '--check-only' +// returns a non-nil error when a generated commit TOML is stale without writing. +func TestRefreshUpstreamCommits_CheckOnly_StaleReturnsError(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + const initialCommit = "initial-aaa111" + + const advancedCommit = "advanced-bbb222" + + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + preStore := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, preStore.Save("curl", initialCommit)) + + addRefreshUpstreamComponent(env, "curl") + setupMockGit(env, advancedCommit) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + CheckOnly: true, + }) + require.Error(t, err, "stale TOML must produce a non-nil error in --check-only mode") + assert.Contains(t, err.Error(), "stale", "error message should mention staleness") + assert.Contains(t, err.Error(), "curl", "error message should name the stale component") + assert.Contains(t, err.Error(), "azldev --without-lockfile component refresh-upstream-commit -a", + "-a-scoped run should suggest the same -a invocation to refresh") + + // Results slice must be returned alongside the error so structured + // consumers (e.g. -O json) retain per-component data on stale runs. + require.NotEmpty(t, results, "results must be returned even when stale") + + var foundCurl bool + + for _, r := range results { + if r.Component == "curl" { + foundCurl = true + + assert.True(t, r.Changed, "stale curl must surface as Changed in returned results") + } + } + + assert.True(t, foundCurl, "stale curl must appear in returned results slice") + + freshStore := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + savedCommit, exists, loadErr := freshStore.Get("curl") + require.NoError(t, loadErr) + assert.True(t, exists) + assert.Equal(t, initialCommit, savedCommit) +} + +// TestRefreshUpstreamCommits_CheckOnly_FreshReturnsNil verifies that '--check-only' +// returns nil when all generated commit TOMLs are fresh. +func TestRefreshUpstreamCommits_CheckOnly_FreshReturnsNil(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + const commit = "fresh-commit-aaa" + + setupMockGit(env, commit) + addRefreshUpstreamComponent(env, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + + options := &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + } + + // Phase 1: populate the generated TOML with a real refresh run. + _, err := componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err) + + freshStore := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + before, beforeExists, loadErr := freshStore.Get("curl") + require.NoError(t, loadErr) + require.True(t, beforeExists) + + // Phase 2: --check-only against the now-fresh TOML. Must return nil. + options.CheckOnly = true + _, err = componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err, "fresh TOMLs must return nil error in --check-only mode") + + // The configured commit must remain unchanged. + freshStore = upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + after, afterExists, loadErr := freshStore.Get("curl") + require.NoError(t, loadErr) + require.True(t, afterExists) + assert.Equal(t, before, after) +} + +// TestRefreshUpstreamCommits_CheckOnly_DetectsOrphans verifies that '--check-only' +// returns an error when an orphan generated TOML would be pruned by a normal run, +// and that the orphan is NOT actually deleted. +func TestRefreshUpstreamCommits_CheckOnly_DetectsOrphans(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + + const commit = "fresh-commit-aaa" + + setupMockGit(env, commit) + addRefreshUpstreamComponent(env, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, testUpstreamCommitsDir)) + + // First, do a real refresh so curl's TOML is fresh; this isolates the orphan as + // the only thing --check-only should flag. + _, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + + // Plant an orphan TOML after the refresh; a normal refresh would have + // pruned it. The orphan does NOT correspond to any component in config. + preStore := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, preStore.Save("removed-pkg", "orphan-commit")) + + // --check-only must report the orphan and not delete it. + _, err = componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + CheckOnly: true, + }) + require.Error(t, err, "orphan TOML must produce an error in --check-only mode") + assert.Contains(t, err.Error(), "orphan") + assert.Contains(t, err.Error(), "removed-pkg") + + freshStore := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + _, exists, loadErr := freshStore.Get("removed-pkg") + require.NoError(t, loadErr) + assert.True(t, exists, "--check-only must not prune orphan TOMLs") +} diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 163da0dd9..644c9bcfb 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -38,12 +38,12 @@ type RenderOptions struct { CheckOnly bool } -func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewRenderCmd()) +func renderOnAppInit(app *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewRenderCmd(cmdOptionsForApp(app)...)) } // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. -func NewRenderCmd() *cobra.Command { +func NewRenderCmd(opts ...CmdOption) *cobra.Command { var options RenderOptions var cmd *cobra.Command @@ -97,7 +97,7 @@ valid with -a.`, }, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", "output directory for rendered specs (overrides rendered-specs-dir from config)") @@ -523,13 +523,11 @@ func prepareComponentSources( // rpmautospec can expand %autorelease and %autochangelog correctly. // WithSkipLookaside avoids expensive tarball downloads — only spec + // sidecar files are needed for rendering. - preparerOpts := []sources.PreparerOption{ - sources.WithGitRepo(env, env.LockReader(), distro.Version.ReleaseVer), - sources.WithDirtyDetection(), + preparerOpts := append(gitRepoPreparerOptions(env, distro), sources.WithSkipLookaside(), sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), sources.WithMockProcessor(mockProcessor), - } + ) preparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env, preparerOpts...) if err != nil { diff --git a/internal/app/azldev/cmds/component/update.go b/internal/app/azldev/cmds/component/update.go index 8e003d24c..8de54c317 100644 --- a/internal/app/azldev/cmds/component/update.go +++ b/internal/app/azldev/cmds/component/update.go @@ -96,7 +96,7 @@ Cannot be combined with --bump.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter) + addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) cmd.Flags().BoolVar(&options.Bump, "bump", false, "increment the manual-rebuild counter to trigger a new release") @@ -256,42 +256,43 @@ func handleOrphanLocks( comps []components.Component, options *UpdateComponentOptions, ) ([]string, error) { - if !options.ComponentFilter.IncludeAllComponents { - return nil, nil - } - - if len(comps) == 0 { - if options.CheckOnly { - slog.Warn("No components resolved; all existing lock files would be treated as orphans") - } else { - slog.Warn("No components resolved; all existing lock files will be treated as orphans") - } - } - - resolvedNames := make(map[string]projectconfig.ComponentConfig, len(comps)) - for _, comp := range comps { - resolvedNames[comp.GetName()] = *comp.GetConfig() - } + return handleOrphans( + lockOrphanStore{store: store}, + comps, + options.ComponentFilter.IncludeAllComponents, + options.CheckOnly, + orphanMessages{ + noComponents: "all existing lock files", + pruned: "Pruned orphan lock files", + }, + ) +} - if options.CheckOnly { - orphans, findErr := store.FindOrphanLockFiles(resolvedNames) - if findErr != nil { - return nil, fmt.Errorf("finding orphan lock files:\n%w", findErr) - } +// lockOrphanStore adapts [lockfile.Store] to [orphanStore]. +type lockOrphanStore struct { + store *lockfile.Store +} - return orphans, nil +func (s lockOrphanStore) findOrphans( + resolved map[string]projectconfig.ComponentConfig, +) ([]string, error) { + orphans, err := s.store.FindOrphanLockFiles(resolved) + if err != nil { + return nil, fmt.Errorf("finding orphan lock files:\n%w", err) } - pruned, pruneErr := store.PruneOrphans(resolvedNames) - if pruneErr != nil { - return nil, fmt.Errorf("pruning orphan lock files:\n%w", pruneErr) - } + return orphans, nil +} - if pruned > 0 { - slog.Info("Pruned orphan lock files", "count", pruned) +func (s lockOrphanStore) pruneOrphans( + resolved map[string]projectconfig.ComponentConfig, +) (int, error) { + pruned, err := s.store.PruneOrphans(resolved) + if err != nil { + return 0, fmt.Errorf("pruning orphan lock files:\n%w", err) } - return nil, nil + return pruned, nil } // checkOnlyResult inspects the results of a --check-only update run and diff --git a/internal/app/azldev/cmds/component/update_test.go b/internal/app/azldev/cmds/component/update_test.go index 57dea6449..5a2740dcf 100644 --- a/internal/app/azldev/cmds/component/update_test.go +++ b/internal/app/azldev/cmds/component/update_test.go @@ -445,7 +445,7 @@ func TestUpdateComponents_CheckOnly_StaleReturnsError(t *testing.T) { var foundCurl bool for _, r := range results { - if r.Component == "curl" { + if r.Component == testComponentName { foundCurl = true assert.True(t, r.Changed, "stale curl must surface as Changed in returned results") diff --git a/internal/app/azldev/cmds/docs/agent.go b/internal/app/azldev/cmds/docs/agent.go index a3ab5e415..00cfad9bb 100644 --- a/internal/app/azldev/cmds/docs/agent.go +++ b/internal/app/azldev/cmds/docs/agent.go @@ -13,6 +13,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/agentskill" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/upstreamcommit" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/spf13/cobra" @@ -34,11 +35,11 @@ type InstalledAgentFile struct { } // Called once when the app is initialized; registers the 'agent' command tree under 'docs'. -func agentOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(newAgentCmd()) +func agentOnAppInit(app *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(newAgentCmd(agentskill.NewCatalog(app.WithoutLockfile()))) } -func newAgentCmd() *cobra.Command { +func newAgentCmd(catalog agentskill.Catalog) *cobra.Command { cmd := &cobra.Command{ Use: "agent", Short: "Emit AI agent skill and instruction files", @@ -51,7 +52,7 @@ reference so that agents always load the guidance that ships with the binary.`, } cmd.AddCommand(newAgentInstallCmd()) - cmd.AddCommand(newAgentShowCmd()) + cmd.AddCommand(newAgentShowCmd(catalog)) return cmd } @@ -81,11 +82,13 @@ read-only 'docs-agent-show' MCP tool for the full, always-current skill. Pass --full to inline the complete skill instead, for environments without the azldev MCP server. -Directory paths in the emitted content (such as the lock and rendered-spec -directories) are resolved from the loaded azldev.toml, falling back to azldev's -built-in defaults when no configuration is found. The bindings reflect the project -azldev runs in, so pair --output-dir with -C pointing at the target repository when -scaffolding a different repo.`, +Directory paths in the emitted content (such as the lock, generated +upstream-commit, and rendered-spec directories) are resolved from the loaded +azldev.toml, falling back to azldev's built-in defaults when no configuration is +found. The emitted content also reflects the mode azldev runs in, so pass +--without-lockfile to describe the lock-file-free workflow. The bindings reflect +the project azldev runs in, so pair --output-dir with -C pointing at the target +repository when scaffolding a different repo.`, Example: ` # Write agent files into the current repository azldev docs agent install @@ -113,10 +116,10 @@ scaffolding a different repo.`, return cmd } -func newAgentShowCmd() *cobra.Command { +func newAgentShowCmd(catalog agentskill.Catalog) *cobra.Command { var skillName string - completeSkillNames := cobra.FixedCompletions(skillNames(), cobra.ShellCompDirectiveNoFileComp) + completeSkillNames := cobra.FixedCompletions(skillNames(catalog), cobra.ShellCompDirectiveNoFileComp) cmd := &cobra.Command{ Use: "show", @@ -139,7 +142,12 @@ skill to list the available skills.`, } cmd.RunE = func(cmd *cobra.Command, _ []string) error { - name, list, err := resolveShowSkill(skillName) + env, err := azldev.GetEnvFromCommand(cmd) + if err != nil { + return fmt.Errorf("failed to get command environment:\n%w", err) + } + + name, list, err := resolveShowSkill(agentSkillCatalog(env), skillName) if err != nil { return err } @@ -153,12 +161,7 @@ skill to list the available skills.`, return nil } - env, err := azldev.GetEnvFromCommand(cmd) - if err != nil { - return fmt.Errorf("failed to get command environment:\n%w", err) - } - - doc, err := agentskill.SkillDocument(name, agentSkillParams(env, cmd.Root())) + doc, err := agentSkillCatalog(env).SkillDocument(name, agentSkillParams(env, cmd.Root())) if err != nil { return fmt.Errorf("failed to render azldev skill:\n%w", err) } @@ -186,7 +189,7 @@ func InstallAgentFiles( return nil, err } - files, err := agentskill.Files(layout, agentSkillParams(env, rootCmd), options.Full) + files, err := agentSkillCatalog(env).Files(layout, agentSkillParams(env, rootCmd), options.Full) if err != nil { return nil, fmt.Errorf("failed to render azldev agent files:\n%w", err) } @@ -313,6 +316,12 @@ func emitMCPConfig( return InstalledAgentFile{Path: destPath, Written: true}, nil } +// agentSkillCatalog returns the skill catalog describing the mode azldev is running +// in, so the emitted and served content matches the commands the user actually has. +func agentSkillCatalog(env *azldev.Env) agentskill.Catalog { + return agentskill.NewCatalog(env != nil && env.WithoutLockfile()) +} + // agentSkillParams gathers the dynamic values injected into the emitted and served agent content, // including the target-repo bindings resolved from the loaded project configuration. func agentSkillParams(env *azldev.Env, rootCmd *cobra.Command) agentskill.Params { @@ -330,9 +339,10 @@ func agentSkillParams(env *azldev.Env, rootCmd *cobra.Command) agentskill.Params // with '-C' pointing at that repository so the emitted paths match it. func resolveBindings(env *azldev.Env) agentskill.Bindings { bindings := agentskill.Bindings{ - LockDir: projectconfig.DefaultLockDir, - RenderedSpecsDir: projectconfig.DefaultRenderedSpecsDir, - WorkDir: projectconfig.DefaultWorkDir, + LockDir: projectconfig.DefaultLockDir, + UpstreamCommitsDir: upstreamcommit.DefaultDir, + RenderedSpecsDir: projectconfig.DefaultRenderedSpecsDir, + WorkDir: projectconfig.DefaultWorkDir, } if env == nil || env.Config() == nil { @@ -394,14 +404,14 @@ func resolveLayout(layoutName string) (agentskill.Layout, error) { // It returns the skill name to print; or, when no skill is named, an empty name plus // the list of skill names for the caller to display. // An unknown name is an error that names the valid choices. -func resolveShowSkill(requested string) (name string, list []string, err error) { - names := skillNames() +func resolveShowSkill(catalog agentskill.Catalog, requested string) (name string, list []string, err error) { + names := skillNames(catalog) if requested == "" { return "", names, nil } - if _, findErr := agentskill.FindSkill(requested); findErr != nil { + if _, findErr := catalog.FindSkill(requested); findErr != nil { return "", nil, fmt.Errorf("unknown skill %#q; choose one of: %s", requested, strings.Join(names, ", ")) } @@ -409,9 +419,9 @@ func resolveShowSkill(requested string) (name string, list []string, err error) return requested, nil, nil } -// skillNames returns the registered skill names in emission order. -func skillNames() []string { - skills := agentskill.Skills() +// skillNames returns the catalog's skill names in emission order. +func skillNames(catalog agentskill.Catalog) []string { + skills := catalog.Skills() names := make([]string, len(skills)) for i, skill := range skills { diff --git a/internal/app/azldev/cmds/docs/agent_internal_test.go b/internal/app/azldev/cmds/docs/agent_internal_test.go index 2ae75804b..6e9ddf32c 100644 --- a/internal/app/azldev/cmds/docs/agent_internal_test.go +++ b/internal/app/azldev/cmds/docs/agent_internal_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/agentskill" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/spf13/cobra" @@ -71,29 +72,34 @@ func TestResolveBindingsFromConfig(t *testing.T) { assert.Equal(t, "build/work", bindings.WorkDir) } +// defaultCatalog is the agent skill catalog for azldev's default mode. +// +//nolint:gochecknoglobals // effectively a constant used by several tests. +var defaultCatalog = agentskill.NewCatalog(false) + func TestResolveShowSkill(t *testing.T) { // A known skill resolves to itself, with nothing to list. - name, list, err := resolveShowSkill("azldev") + name, list, err := resolveShowSkill(defaultCatalog, "azldev") require.NoError(t, err) assert.Equal(t, "azldev", name) assert.Nil(t, list) // No skill named, with several registered, returns the names to display. - name, list, err = resolveShowSkill("") + name, list, err = resolveShowSkill(defaultCatalog, "") require.NoError(t, err) assert.Empty(t, name) - assert.Equal(t, skillNames(), list) + assert.Equal(t, skillNames(defaultCatalog), list) assert.Greater(t, len(list), 1, "test assumes more than one skill is registered") // An unknown skill errors and names the valid choices. - _, _, err = resolveShowSkill("not-a-real-skill") + _, _, err = resolveShowSkill(defaultCatalog, "not-a-real-skill") require.Error(t, err) assert.Contains(t, err.Error(), "unknown skill") assert.Contains(t, err.Error(), "azldev") } func TestAgentShowCmdUsesSkillFlag(t *testing.T) { - cmd := newAgentShowCmd() + cmd := newAgentShowCmd(defaultCatalog) assert.Equal(t, "show", cmd.Use) require.NoError(t, cmd.Args(cmd, nil)) @@ -103,6 +109,6 @@ func TestAgentShowCmdUsesSkillFlag(t *testing.T) { require.True(t, ok, "--skill must have shell completion") choices, directive := complete(cmd, nil, "") - assert.Equal(t, skillNames(), choices) + assert.Equal(t, skillNames(defaultCatalog), choices) assert.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) } diff --git a/internal/app/azldev/cmds/docs/markdown.go b/internal/app/azldev/cmds/docs/markdown.go index 37b17510c..2715a1003 100644 --- a/internal/app/azldev/cmds/docs/markdown.go +++ b/internal/app/azldev/cmds/docs/markdown.go @@ -99,6 +99,37 @@ func setHiddenRecursive(cmd *cobra.Command, hidden bool) { } } +// commandHiddenStates records the Hidden flag of cmd and all of its descendants so +// that temporary changes made during generation can be reverted. +func commandHiddenStates(cmd *cobra.Command) map[*cobra.Command]bool { + states := make(map[*cobra.Command]bool) + + var collect func(*cobra.Command) + + collect = func(current *cobra.Command) { + states[current] = current.Hidden + for _, child := range current.Commands() { + collect(child) + } + } + collect(cmd) + + return states +} + +// hideMarkdownExcludedCommands hides every command annotated with +// [azldev.CmdAnnotationMarkdownDocsExcluded], even when hidden commands are +// otherwise being documented. +func hideMarkdownExcludedCommands(cmd *cobra.Command) { + if _, excluded := cmd.Annotations[azldev.CmdAnnotationMarkdownDocsExcluded]; excluded { + cmd.Hidden = true + } + + for _, child := range cmd.Commands() { + hideMarkdownExcludedCommands(child) + } +} + // CheckOutputDir verifies the output directory state before generation. // If the directory exists and is non-empty, it either removes it (when Force is set) // or returns an actionable error suggesting --force / -f. @@ -140,12 +171,17 @@ func GenerateMarkdownDocs(fs opctx.FS, rootCmd *cobra.Command, options *Generate // Strip the dynamic version string from the root command's Short description and disable // the auto-generated date footer so that generated docs don't churn on every build. origShort := rootCmd.Short + origHiddenStates := commandHiddenStates(rootCmd) rootCmd.Short = stripVersionFromShort(origShort) rootCmd.DisableAutoGenTag = true defer func() { rootCmd.Short = origShort rootCmd.DisableAutoGenTag = false + + for cmd, hidden := range origHiddenStates { + cmd.Hidden = hidden + } }() // Check the output directory state and handle force semantics. @@ -165,6 +201,8 @@ func GenerateMarkdownDocs(fs opctx.FS, rootCmd *cobra.Command, options *Generate setHiddenRecursive(rootCmd, false) } + hideMarkdownExcludedCommands(rootCmd) + // NOTE: This function can't work with our [opctx.FS] filesystem abstraction, but there's not much // we can do about that. err = doc.GenMarkdownTreeCustom(rootCmd, options.OutputDir, filePrepender, linkHandler) diff --git a/internal/app/azldev/command.go b/internal/app/azldev/command.go index 8780e2e32..54c326c8d 100644 --- a/internal/app/azldev/command.go +++ b/internal/app/azldev/command.go @@ -20,6 +20,9 @@ const ( // CommandAnnotationRootOK is a [cobra.Command.Annotations] key used to indicate that a command // is allowed to be run as root. CommandAnnotationRootOK = "rootOK" + // CommandAnnotationPermissiveConfig is a [cobra.Command.Annotations] key used to indicate that + // project configuration must be loaded permissively before running a command. + CommandAnnotationPermissiveConfig = "azldev.config.permissive" ) const ( @@ -41,6 +44,10 @@ const CmdAnnotationMCPEnabled = "azldev.mcp.enabled" // hint to auto-approve the tool. The value associated with the key is ignored. const CmdAnnotationMCPReadOnly = "azldev.mcp.readonly" +// CmdAnnotationMarkdownDocsExcluded prevents a command from appearing in generated Markdown +// reference documentation, including when hidden commands are requested. +const CmdAnnotationMarkdownDocsExcluded = "azldev.docs.markdown-excluded" + // cmdMCPAnnotationValue is the placeholder value stored for MCP command annotations; only the // presence of the key matters. const cmdMCPAnnotationValue = "true" @@ -243,6 +250,16 @@ func ExportAsReadOnlyMCPTool(cmd *cobra.Command) { } } +// ExcludeFromMarkdownDocs prevents cmd from appearing in generated Markdown reference +// documentation while leaving it registered for compatibility. +func ExcludeFromMarkdownDocs(cmd *cobra.Command) { + if cmd.Annotations == nil { + cmd.Annotations = make(map[string]string) + } + + cmd.Annotations[CmdAnnotationMarkdownDocsExcluded] = "true" +} + // Displays the results of a command in the appropriate format to stdout. func reportResults(env *Env, results interface{}) error { switch env.defaultReportFormat { diff --git a/internal/app/azldev/core/components/filter.go b/internal/app/azldev/core/components/filter.go index 95c9b5821..0b29c803b 100644 --- a/internal/app/azldev/core/components/filter.go +++ b/internal/app/azldev/core/components/filter.go @@ -52,7 +52,12 @@ func AddComponentFilterOptionsToCommand(cmd *cobra.Command, filter *ComponentFil cmd.Flags().StringArrayVarP(&filter.SpecPaths, "spec-path", "s", []string{}, "Spec path") _ = cmd.MarkFlagFilename("spec-path", ".spec") +} +// AddLockValidationFlagToCommand adds the '--skip-lock-validation' flag to a command. +// Only azldev's default mode validates lock files, so lock-file-free mode leaves the +// flag unregistered rather than accepting an option that does nothing. +func AddLockValidationFlagToCommand(cmd *cobra.Command, filter *ComponentFilter) { cmd.Flags().BoolVar(&filter.SkipLockValidation, "skip-lock-validation", false, "skip lock file consistency checks") diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 4d32c9834..f9694a1d4 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -98,6 +98,16 @@ func WithDirtyDetection() PreparerOption { } } +// WithoutLockfileHistory returns a [PreparerOption] that derives synthetic history +// from the component's generated upstream-commit TOML instead of its lock file. +// It selects the behavior of the global '--without-lockfile' flag; the lock reader +// and release version passed to [WithGitRepo] are unused in that mode. +func WithoutLockfileHistory() PreparerOption { + return func(p *sourcePreparerImpl) { + p.withoutLockfile = true + } +} + // WithSkipLookaside returns a [PreparerOption] that skips all lookaside cache // downloads during source preparation. This includes both explicit source file // downloads ([SourceManager.FetchFiles]) and lookaside extraction during @@ -184,6 +194,11 @@ type sourcePreparerImpl struct { // synthetic history generation. Set via [WithDirtyDetection]. dirtyDetection bool + // withoutLockfile, when true, derives synthetic history from the generated + // upstream-commit TOML rather than the lock file. Set via + // [WithoutLockfileHistory]. + withoutLockfile bool + // releaseVer is the per-component resolved distro release version, not the // project default. Set via [WithGitRepo]. releaseVer string @@ -244,6 +259,11 @@ func NewPreparer( "dirty detection compares fingerprints against committed lock files in the git history") } + if impl.dirtyDetection && impl.withoutLockfile { + return nil, errors.New("WithDirtyDetection is incompatible with WithoutLockfileHistory; " + + "there is no lock file to compare fingerprints against") + } + return impl, nil } @@ -500,26 +520,9 @@ func (p *sourcePreparerImpl) trySyntheticHistory( config := component.GetConfig() componentName := component.GetName() - // Compute the current fingerprint for uncommitted-change detection. - // Only computed when dirty detection is enabled (e.g., build, render). - // An empty fingerprint skips dirty detection in buildSyntheticCommits. - var currentFingerprint string - - if p.dirtyDetection { - var fpErr error - - currentFingerprint, fpErr = computeCurrentFingerprint(p.fs, fingerprintConfig, p.releaseVer) - if fpErr != nil { - return fmt.Errorf("dirty detection failed for component %#q:\n%w", componentName, fpErr) - } - } - - changes, importCommit, err := buildSyntheticCommits( - ctx, p.cmdFactory, config, componentName, p.lockReader.LockDir(), - currentFingerprint, - ) + changes, importCommit, err := p.findSyntheticChanges(ctx, config, fingerprintConfig, componentName) if err != nil { - return fmt.Errorf("failed to build synthetic commits:\n%w", err) + return err } if len(changes) == 0 { @@ -570,6 +573,52 @@ func (p *sourcePreparerImpl) trySyntheticHistory( return nil } +// findSyntheticChanges discovers the component changes that synthetic history +// should represent, using the source selected by the preparer's mode: the +// component's lock file by default, or its generated upstream-commit TOML when +// [WithoutLockfileHistory] is set. The returned import commit bounds the upstream +// walk; it is always empty in lock-file-free mode, where no fork point is +// persisted and the repository's first-parent root bounds the walk instead. +func (p *sourcePreparerImpl) findSyntheticChanges( + ctx context.Context, + config *projectconfig.ComponentConfig, + fingerprintConfig *projectconfig.ComponentConfig, + componentName string, +) (changes []FingerprintChange, importCommit string, err error) { + if p.withoutLockfile { + changes, err = buildUpstreamCommitSyntheticCommits(ctx, p.cmdFactory, config, componentName) + if err != nil { + return nil, "", fmt.Errorf("failed to build synthetic commits:\n%w", err) + } + + return changes, "", nil + } + + // Compute the current fingerprint for uncommitted-change detection. + // Only computed when dirty detection is enabled (e.g., build, render). + // An empty fingerprint skips dirty detection in buildSyntheticCommits. + var currentFingerprint string + + if p.dirtyDetection { + var fpErr error + + currentFingerprint, fpErr = computeCurrentFingerprint(p.fs, fingerprintConfig, p.releaseVer) + if fpErr != nil { + return nil, "", fmt.Errorf("dirty detection failed for component %#q:\n%w", componentName, fpErr) + } + } + + changes, importCommit, err = buildSyntheticCommits( + ctx, p.cmdFactory, config, componentName, p.lockReader.LockDir(), + currentFingerprint, + ) + if err != nil { + return nil, "", fmt.Errorf("failed to build synthetic commits:\n%w", err) + } + + return changes, importCommit, nil +} + // computeCurrentFingerprint computes the current input fingerprint for a // component from its resolved config. Returns ("", nil) for local components // or when the source identity cannot be determined — dirty detection is diff --git a/internal/app/azldev/core/sources/synthistory.go b/internal/app/azldev/core/sources/synthistory.go index 9d66f152d..ae0467eef 100644 --- a/internal/app/azldev/core/sources/synthistory.go +++ b/internal/app/azldev/core/sources/synthistory.go @@ -42,6 +42,12 @@ type FingerprintChange struct { UpstreamCommit string } +// UpstreamCommitChange records a project commit that changed a component's +// configured upstream commit. Lock-file-free mode discovers changes from the +// generated upstream-commit TOML instead of a lock file's fingerprint, but the +// recorded data and the replay that consumes it are identical. +type UpstreamCommitChange = FingerprintChange + // interleavedEntry represents a single commit in the rebuilt dist-git history. // Exactly one of upstreamCommit or syntheticChange is non-nil. type interleavedEntry struct { @@ -585,15 +591,29 @@ func openProjectRepo( config *projectconfig.ComponentConfig, componentName string, ) (*gogit.Repository, string, error) { - if config.SourceConfigFile == nil || config.SourceConfigFile.SourcePath() == "" { + var configFilePath string + if config.SourceConfigFile != nil { + configFilePath = config.SourceConfigFile.SourcePath() + } + + return openProjectRepoForConfigFile(configFilePath, componentName) +} + +// openProjectRepoForConfigFile opens the git repository containing configFilePath +// and returns both the [gogit.Repository] and the worktree root directory. Returns +// (nil, "", nil) when the path is empty, indicating that synthetic commits should +// be skipped. +func openProjectRepoForConfigFile( + configFilePath string, + componentName string, +) (*gogit.Repository, string, error) { + if configFilePath == "" { slog.Debug("Cannot resolve config file for synthetic commits; skipping", "component", componentName) return nil, "", nil } - configFilePath := config.SourceConfigFile.SourcePath() - repo, err := git.OpenProjectRepo(filepath.Dir(configFilePath)) if err != nil { return nil, "", fmt.Errorf("failed to find project repository for config file %#q:\n%w", diff --git a/internal/app/azldev/core/sources/synthistory_upstreamcommit.go b/internal/app/azldev/core/sources/synthistory_upstreamcommit.go new file mode 100644 index 000000000..bbad7f5fb --- /dev/null +++ b/internal/app/azldev/core/sources/synthistory_upstreamcommit.go @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "context" + "errors" + "fmt" + "log/slog" + "path/filepath" + "slices" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + toml "github.com/pelletier/go-toml/v2" +) + +// This file implements synthetic-history discovery for lock-file-free mode. It +// derives the change history from the generated upstream-commit TOML that pins a +// component's commit, rather than from lock file fingerprint changes. Everything +// downstream of discovery (interleaving and replay) is shared with the default +// mode; see synthistory.go. + +// FindUpstreamCommitChanges walks the git log for commits that changed a +// component's generated upstream-commit TOML. Results are chronological +// (oldest first). +func FindUpstreamCommitChanges( + ctx context.Context, + cmdFactory opctx.CmdFactory, + projectRepo *gogit.Repository, + projectRepoDir string, + configFileRelPath string, + componentName string, +) ([]UpstreamCommitChange, error) { + metas, err := gitLogFileMetadata(ctx, cmdFactory, projectRepoDir, configFileRelPath) + if err != nil { + return nil, err + } + + if len(metas) == 0 { + return nil, nil + } + + type entry struct { + upstreamCommit string + meta CommitMetadata + } + + var entries []entry //nolint:prealloc // size not known ahead of time. + + for _, meta := range metas { + upstreamCommit, err := upstreamCommitAtCommit( + projectRepo, meta.Hash, configFileRelPath, componentName, + ) + if errors.Is(err, object.ErrFileNotFound) || errors.Is(err, object.ErrDirectoryNotFound) { + // The commit deleted the generated TOML; the pin it carried is the + // one recorded by its parent. + upstreamCommit, err = upstreamCommitBeforeCommit( + projectRepo, meta.Hash, configFileRelPath, componentName, + ) + } + + if err != nil { + return nil, fmt.Errorf("failed to read upstream commit TOML at commit %#q:\n%w", + meta.Hash, err) + } + + entries = append(entries, entry{upstreamCommit: upstreamCommit, meta: meta}) + } + + if len(entries) == 0 { + return nil, nil + } + + // Entries are newest-first (from git log order). Reverse to chronological. + slices.Reverse(entries) + + changes := make([]UpstreamCommitChange, 0, len(entries)) + for _, change := range entries { + changes = append(changes, UpstreamCommitChange{ + CommitMetadata: change.meta, + UpstreamCommit: change.upstreamCommit, + }) + } + + return changes, nil +} + +// upstreamCommitBeforeCommit reads the component's pinned upstream commit from the +// first parent of commitHash. +func upstreamCommitBeforeCommit( + repo *gogit.Repository, + commitHash string, + configFileRelPath string, + componentName string, +) (string, error) { + commit, err := repo.CommitObject(plumbing.NewHash(commitHash)) + if err != nil { + return "", fmt.Errorf("failed to resolve config deletion commit %#q:\n%w", + commitHash, err) + } + + parent, err := commit.Parent(0) + if err != nil { + return "", fmt.Errorf("failed to resolve parent of config deletion commit %#q:\n%w", + commitHash, err) + } + + return upstreamCommitAtCommit(repo, parent.Hash.String(), configFileRelPath, componentName) +} + +// upstreamCommitAtCommit reads the component's pinned upstream commit from the +// generated TOML as it existed at commitHash. +func upstreamCommitAtCommit( + repo *gogit.Repository, + commitHash string, + configFileRelPath string, + componentName string, +) (string, error) { + commit, err := repo.CommitObject(plumbing.NewHash(commitHash)) + if err != nil { + return "", fmt.Errorf("resolving commit %#q:\n%w", commitHash, err) + } + + tree, err := commit.Tree() + if err != nil { + return "", fmt.Errorf("reading commit tree:\n%w", err) + } + + file, err := tree.File(configFileRelPath) + if err != nil { + return "", fmt.Errorf("reading config file %#q:\n%w", configFileRelPath, err) + } + + content, err := file.Contents() + if err != nil { + return "", fmt.Errorf("reading config contents %#q:\n%w", configFileRelPath, err) + } + + var config projectconfig.ConfigFile + if err := toml.Unmarshal([]byte(content), &config); err != nil { + return "", fmt.Errorf("parsing config file:\n%w", err) + } + + component, ok := config.Components[componentName] + if !ok { + return "", fmt.Errorf("config file does not define component %#q", componentName) + } + + return component.Spec.UpstreamCommit, nil +} + +// buildUpstreamCommitSyntheticCommits resolves the project repository from the +// config file that pinned the component's upstream commit and returns that file's +// upstream-commit changes chronologically. Returns (nil, nil) when there is no +// history to represent. +func buildUpstreamCommitSyntheticCommits( + ctx context.Context, + cmdFactory opctx.CmdFactory, + config *projectconfig.ComponentConfig, + componentName string, +) ([]UpstreamCommitChange, error) { + var configFileAbsPath string + if configFile := config.UpstreamCommitConfigFile(); configFile != nil { + configFileAbsPath = configFile.SourcePath() + } + + projectRepo, projectRepoDir, err := openProjectRepoForConfigFile(configFileAbsPath, componentName) + if err != nil { + return nil, err + } + + if projectRepo == nil { + return nil, nil + } + + configFileRelPath, err := filepath.Rel(projectRepoDir, configFileAbsPath) + if err != nil { + return nil, fmt.Errorf("failed to compute repo-relative config path for %#q:\n%w", + configFileAbsPath, err) + } + + if config.Spec.UpstreamCommit == "" { + return nil, nil + } + + changes, err := FindUpstreamCommitChanges( + ctx, cmdFactory, projectRepo, projectRepoDir, configFileRelPath, componentName, + ) + if err != nil { + return nil, fmt.Errorf("failed to find upstream commit changes for config file %#q:\n%w", + configFileRelPath, err) + } + + if len(changes) == 0 { + shallowCommits, _ := projectRepo.Storer.Shallow() + if len(shallowCommits) > 0 { + return nil, fmt.Errorf( + "upstream commit TOML %#q has no git history; a full clone is required", + configFileRelPath) + } + + slog.Warn("Upstream commit TOML has no changes; skipping synthetic history", + "configFile", configFileRelPath) + + return nil, nil + } + + return changes, nil +} diff --git a/internal/app/azldev/core/sources/synthistory_upstreamcommit_internal_test.go b/internal/app/azldev/core/sources/synthistory_upstreamcommit_internal_test.go new file mode 100644 index 000000000..0a156fdb8 --- /dev/null +++ b/internal/app/azldev/core/sources/synthistory_upstreamcommit_internal_test.go @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "testing" + "time" + + "github.com/go-git/go-billy/v5" + memfs "github.com/go-git/go-billy/v5/memfs" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeGeneratedCommit commits a generated upstream-commit TOML holding the given +// pin and returns the resulting commit hash. +func writeGeneratedCommit( + t *testing.T, repo *gogit.Repository, memFS billy.Filesystem, relPath, upstreamCommit string, +) string { + t.Helper() + + worktree, err := repo.Worktree() + require.NoError(t, err) + + file, err := memFS.Create(relPath) + require.NoError(t, err) + + _, err = file.Write([]byte("[components.curl.spec]\nupstream-commit = \"" + upstreamCommit + "\"\n")) + require.NoError(t, err) + require.NoError(t, file.Close()) + + _, err = worktree.Add(relPath) + require.NoError(t, err) + + hash, err := worktree.Commit("pin "+upstreamCommit, &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "azldev", + Email: "azldev@local", + When: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + }, + }) + require.NoError(t, err) + + return hash.String() +} + +func TestUpstreamCommitAtCommit(t *testing.T) { + memFS := memfs.New() + + repo, err := gogit.Init(memory.NewStorage(), memFS) + require.NoError(t, err) + + const relPath = "base/upstream-commits/curl.toml" + + first := writeGeneratedCommit(t, repo, memFS, relPath, "aaa1111") + second := writeGeneratedCommit(t, repo, memFS, relPath, "bbb2222") + + firstPin, err := upstreamCommitAtCommit(repo, first, relPath, "curl") + require.NoError(t, err) + assert.Equal(t, "aaa1111", firstPin) + + secondPin, err := upstreamCommitAtCommit(repo, second, relPath, "curl") + require.NoError(t, err) + assert.Equal(t, "bbb2222", secondPin) + + // The pin recorded before a commit comes from that commit's first parent. + previousPin, err := upstreamCommitBeforeCommit(repo, second, relPath, "curl") + require.NoError(t, err) + assert.Equal(t, "aaa1111", previousPin) +} + +func TestUpstreamCommitAtCommit_UnknownComponent(t *testing.T) { + memFS := memfs.New() + + repo, err := gogit.Init(memory.NewStorage(), memFS) + require.NoError(t, err) + + const relPath = "base/upstream-commits/curl.toml" + + hash := writeGeneratedCommit(t, repo, memFS, relPath, "aaa1111") + + _, err = upstreamCommitAtCommit(repo, hash, relPath, "openssl") + require.Error(t, err) + assert.Contains(t, err.Error(), "openssl") +} + +func TestFindSyntheticChanges_WithoutLockfile_NoConfigFile(t *testing.T) { + // A component whose upstream commit is not pinned by any config file has no + // generated history to replay, so discovery is skipped rather than failing. + preparer := &sourcePreparerImpl{withoutLockfile: true} + config := &projectconfig.ComponentConfig{Name: "curl"} + + changes, importCommit, err := preparer.findSyntheticChanges(t.Context(), config, config, "curl") + require.NoError(t, err) + assert.Empty(t, changes) + assert.Empty(t, importCommit) +} diff --git a/internal/app/azldev/core/testutils/testenv.go b/internal/app/azldev/core/testutils/testenv.go index 22c08448f..cf913c8ff 100644 --- a/internal/app/azldev/core/testutils/testenv.go +++ b/internal/app/azldev/core/testutils/testenv.go @@ -68,6 +68,21 @@ func NewTestEnv(t *testing.T) *TestEnv { return testEnv } +// NewTestEnvWithoutLockfile creates a [TestEnv] configured for azldev's +// lock-file-free mode: no lock directory is configured and the environment +// reports [azldev.Env.WithoutLockfile], so lock population and lock validation +// are skipped exactly as they are under the global '--without-lockfile' flag. +func NewTestEnvWithoutLockfile(t *testing.T) *TestEnv { + t.Helper() + + testEnv := NewTestEnv(t) + testEnv.Config.Project.LockDir = "" + + setEnvWithOptions(t, testEnv, testEnv.Env.ProjectDir(), true /*withoutLockfile*/) + + return testEnv +} + // newTestEnv creates a new [TestEnv] with a test project config // and mock implementations of [opctx.FS] and [opctx.OSEnv]. func newTestEnv(testMockConfigPath string) *TestEnv { @@ -125,12 +140,21 @@ func populateTestProjectFiles(t *testing.T, testEnv *TestEnv, testProjectDir str func setEnv(t *testing.T, testEnv *TestEnv, testProjectDir string) { t.Helper() + setEnvWithOptions(t, testEnv, testProjectDir, false /*withoutLockfile*/) +} + +// setEnvWithOptions sets the [azldev.Env] for the test environment in the +// requested mode. +func setEnvWithOptions(t *testing.T, testEnv *TestEnv, testProjectDir string, withoutLockfile bool) { + t.Helper() + envOptions := azldev.NewEnvOptions() envOptions.DryRunnable = testEnv.DryRunnable envOptions.EventListener = testEnv.EventListener envOptions.Interfaces = testEnv.TestInterfaces envOptions.ProjectDir = testProjectDir envOptions.Config = testEnv.Config + envOptions.WithoutLockfile = withoutLockfile testEnv.Env = azldev.NewEnv(t.Context(), envOptions) } diff --git a/internal/app/azldev/env.go b/internal/app/azldev/env.go index b1d79acb6..55d7c2d67 100644 --- a/internal/app/azldev/env.go +++ b/internal/app/azldev/env.go @@ -33,6 +33,12 @@ type EnvOptions struct { // The loaded configuration for the project. Config *projectconfig.ProjectConfig + // WithoutLockfile selects the preview lock-file-free mode, in which + // component state is tracked by generated upstream-commit config instead + // of per-component lock files. Set from the global '--without-lockfile' + // flag; false selects the default lock-file behavior. + WithoutLockfile bool + // Injected dependencies. DryRunnable opctx.DryRunnable EventListener opctx.EventListener @@ -73,6 +79,7 @@ type Env struct { acceptAllPrompts bool networkRetries int permissiveConfigParsing bool + withoutLockfile bool // Injected dependencies. cmdFactory opctx.CmdFactory @@ -96,7 +103,8 @@ type Env struct { fixSuggestions *fixSuggestionState // lockStore provides cached access to per-component lock files. - // Nil when no project directory is configured. + // Nil when no project directory is configured, or when lock-file-free + // mode is selected. lockStore *lockfile.Store } @@ -167,6 +175,7 @@ func NewEnv(ctx context.Context, options EnvOptions) *Env { quiet: false, promptsAllowed: isatty.IsTerminal(os.Stdin.Fd()), permissiveConfigParsing: false, + withoutLockfile: options.WithoutLockfile, // Start time. constructionTime: time.Now(), @@ -174,8 +183,9 @@ func NewEnv(ctx context.Context, options EnvOptions) *Env { // No fix suggestions to start. fixSuggestions: &fixSuggestionState{}, - // Lock store: created when we have a project directory. - lockStore: newLockStore(options.ProjectDir, options.Config, options.Interfaces.FileSystemFactory), + // Lock store: created when we have a project directory, unless + // lock-file-free mode is selected. + lockStore: newLockStore(options, options.Interfaces.FileSystemFactory), } } @@ -242,6 +252,13 @@ func (env *Env) SetPermissiveConfigParsing(permissive bool) { env.permissiveConfigParsing = permissive } +// WithoutLockfile reports whether the preview lock-file-free mode is active. +// In that mode azldev tracks resolved upstream commits in generated component +// config instead of per-component lock files, and no lock store is available. +func (env *Env) WithoutLockfile() bool { + return env.withoutLockfile +} + // SetEventListener registers the event listener to be used in this environment. func (env *Env) SetEventListener(eventListener opctx.EventListener) { env.eventListener = eventListener @@ -385,14 +402,16 @@ func (env *Env) LockReader() lockfile.LockReader { } // newLockStore creates a lock store from the project config's lock-dir. -// Returns nil when the project directory, filesystem, or config is unavailable, -// or when the config's lock-dir is empty. +// Returns nil when lock-file-free mode is selected, or when the project +// directory, filesystem, or config is unavailable, or when the config's +// lock-dir is empty. func newLockStore( - projectDir string, - config *projectconfig.ProjectConfig, + options EnvOptions, fsFactory opctx.FileSystemFactory, ) *lockfile.Store { - if projectDir == "" || fsFactory == nil || config == nil || config.Project.LockDir == "" { + config := options.Config + if options.WithoutLockfile || options.ProjectDir == "" || fsFactory == nil || + config == nil || config.Project.LockDir == "" { return nil } diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 763ac0b80..1ededa218 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -368,7 +368,13 @@ type ComponentConfig struct { // Reference to the source config file that this definition came from; not present // in serialized files. - SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-" fingerprint:"-"` + SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-" validate:"-" fingerprint:"-"` + + // upstreamCommitConfigFile references the config file that supplied the + // component's upstream commit pin. Populated in lock-file-free mode, where + // synthetic history must follow the file that actually changed the pin even + // when another partial component definition is merged later. + upstreamCommitConfigFile *ConfigFile // RenderedSpecDir is the output directory for this component's rendered spec files. // Derived at resolve time from the project's rendered-specs-dir setting; not present @@ -458,6 +464,71 @@ func (c *ComponentConfig) MergeUpdatesFrom(other *ComponentConfig) error { return nil } +// MergeOverridesFrom mutates the component config so that values present in other +// replace the existing ones, instead of being merged additively into them. +// +// Used in lock-file-free mode, where a generated upstream-commit config file holds +// a partial component definition whose 'spec' block must override the component's +// primary definition rather than merge with it. Slices in the build config are +// still appended, matching [ComponentConfig.MergeUpdatesFrom]. +func (c *ComponentConfig) MergeOverridesFrom(other *ComponentConfig) error { + otherOverlayFiles := slices.Clone(other.OverlayFiles) + + // Merge the nested config blocks separately so that mergo does not descend + // into them with slice-appending semantics. + otherTopLevel := *other + otherTopLevel.Spec = SpecSource{} + otherTopLevel.Release = ReleaseConfig{} + otherTopLevel.Build = ComponentBuildConfig{} + otherTopLevel.Render = ComponentRenderConfig{} + otherTopLevel.Publish = ComponentPublishConfig{} + + err := mergo.Merge(c, &otherTopLevel, mergo.WithOverride, mergo.WithAppendSlice) + if err != nil { + return fmt.Errorf("failed to merge project info:\n%w", err) + } + + for destination, source := range map[any]any{ + &c.Spec: &other.Spec, + &c.Release: &other.Release, + &c.Render: &other.Render, + &c.Publish: &other.Publish, + } { + if err := mergo.Merge(destination, source, mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge component config:\n%w", err) + } + } + + if err := mergo.Merge(&c.Build, &other.Build, mergo.WithOverride, mergo.WithAppendSlice); err != nil { + return fmt.Errorf("failed to merge component build config:\n%w", err) + } + + if other.SourceConfigFile != nil { + c.SourceConfigFile = other.SourceConfigFile + } + + if other.upstreamCommitConfigFile != nil { + c.upstreamCommitConfigFile = other.upstreamCommitConfigFile + } + + if other.OverlayFiles != nil { + c.OverlayFiles = otherOverlayFiles + } + + return nil +} + +// UpstreamCommitConfigFile returns the config file that supplied the component's +// effective upstream commit pin, or nil when no commit is pinned. Only meaningful +// in lock-file-free mode, where the pin lives in generated component config. +func (c *ComponentConfig) UpstreamCommitConfigFile() *ConfigFile { + if c.upstreamCommitConfigFile == nil && c.Spec.UpstreamCommit != "" { + return c.SourceConfigFile + } + + return c.upstreamCommitConfigFile +} + // EffectiveUpstreamCommit returns the commit to use for upstream operations. // Prefers the locked commit (resolved reality) over the config pin (user intent). // Falls back to Spec.UpstreamCommit for SkipLockValidation paths (update, list, @@ -518,17 +589,18 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi // the SourceConfigFile, as we *do* want to alias that pointer, sharing it across // all configs that came from that source config file. result := &ComponentConfig{ - Name: c.Name, - SourceConfigFile: c.SourceConfigFile, - RenderedSpecDir: c.RenderedSpecDir, - Locked: deep.MustCopy(c.Locked), - Release: c.Release, - Spec: deep.MustCopy(c.Spec), - Build: deep.MustCopy(c.Build), - Render: c.Render, - SourceFiles: deep.MustCopy(c.SourceFiles), - Packages: deep.MustCopy(c.Packages), - Publish: deep.MustCopy(c.Publish), + Name: c.Name, + SourceConfigFile: c.SourceConfigFile, + upstreamCommitConfigFile: c.upstreamCommitConfigFile, + RenderedSpecDir: c.RenderedSpecDir, + Locked: deep.MustCopy(c.Locked), + Release: c.Release, + Spec: deep.MustCopy(c.Spec), + Build: deep.MustCopy(c.Build), + Render: c.Render, + SourceFiles: deep.MustCopy(c.SourceFiles), + Packages: deep.MustCopy(c.Packages), + Publish: deep.MustCopy(c.Publish), // OverlayFiles is consumed after component config resolution; preserve it verbatim // here so inherited patterns can be interpreted relative to the concrete component // config file. diff --git a/internal/projectconfig/config.go b/internal/projectconfig/config.go index a25cf2740..f3d320bef 100644 --- a/internal/projectconfig/config.go +++ b/internal/projectconfig/config.go @@ -17,6 +17,10 @@ import ( // may make use of the provided temporary directory, with the expectation that the caller is responsible // for cleaning it up -- but not until after it is done using the loaded configuration. The loaded // configuration may implicitly depend on the contents of the temporary directory. +// +// When withoutLockfile is set, component definitions merge with override semantics, +// are validated only once the whole project is assembled, and the project's lock +// directory is left unset. func LoadProjectConfig( fs opctx.FS, osEnv opctx.OSEnv, @@ -25,6 +29,7 @@ func LoadProjectConfig( tempDirPath string, extraConfigFilePaths []string, permissiveConfigParsing bool, + withoutLockfile bool, ) (projectDir string, config *ProjectConfig, err error) { // Look for project root and azldev.toml file. projectDir, projectFilePath, err := FindProjectRootAndConfigFile(fs, referenceDir) @@ -79,7 +84,12 @@ func LoadProjectConfig( // // NOTE: We don't wrap the error returned back here (if one is returned) because we already have // a decent one coming from this function. - config, err = loadAndResolveProjectConfig(fs, permissiveConfigParsing, configFilePaths...) + options := loadOptions{ + permissiveConfigParsing: permissiveConfigParsing, + withoutLockfile: withoutLockfile, + } + + config, err = loadAndResolveProjectConfig(fs, options, configFilePaths...) if err != nil { return "", nil, err } @@ -90,5 +100,12 @@ func LoadProjectConfig( // Apply project-relative defaults for any unset path fields. config.Project.ApplyProjectDefaults(projectDir) + // Lock-file-free mode never reads or writes lock files, so the lock directory + // is left unset. A project may still declare 'lock-dir' for compatibility with + // azldev's default mode; the value is simply ignored here. + if withoutLockfile { + config.Project.LockDir = "" + } + return projectDir, config, nil } diff --git a/internal/projectconfig/config_test.go b/internal/projectconfig/config_test.go index d00883148..5912b2c47 100644 --- a/internal/projectconfig/config_test.go +++ b/internal/projectconfig/config_test.go @@ -51,7 +51,7 @@ description = "`+testProjectDesc+`" `) _, config, err := projectconfig.LoadProjectConfig( - ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, + ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, false, ) require.NoError(t, err) require.NotNil(t, config) @@ -66,7 +66,7 @@ func TestLoadProjectConfig_WithDefaultConfig(t *testing.T) { require.NoError(t, fileutils.MkdirAll(ctx.FS(), tempDir)) _, config, err := projectconfig.LoadProjectConfig( - ctx.FS(), ctx.OSEnv(), testProjectDir, false /*disableDefaultConfig*/, tempDir, nil, false, + ctx.FS(), ctx.OSEnv(), testProjectDir, false /*disableDefaultConfig*/, tempDir, nil, false, false, ) require.NoError(t, err) require.NotNil(t, config) @@ -94,7 +94,7 @@ output-dir = "/from/user/out" `), fileperms.PublicFile)) _, config, err := projectconfig.LoadProjectConfig( - ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, + ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, false, ) require.NoError(t, err) require.NotNil(t, config) @@ -136,7 +136,7 @@ description = "`+testUserDesc+`" _, config, err := projectconfig.LoadProjectConfig( ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), - []string{extraConfigPath}, false, + []string{extraConfigPath}, false, false, ) require.NoError(t, err) require.NotNil(t, config) @@ -166,9 +166,41 @@ description = "`+testUserDesc+`" `), fileperms.PublicFile)) _, config, err := projectconfig.LoadProjectConfig( - ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, + ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, false, ) require.NoError(t, err) require.NotNil(t, config) assert.Equal(t, testUserDesc, config.Project.Description) } + +// TestLoadProjectConfig_LockDirByMode verifies that the project's lock directory is +// defaulted in azldev's default mode and left unset in lock-file-free mode, where an +// explicitly configured 'lock-dir' is accepted but ignored. +func TestLoadProjectConfig_LockDirByMode(t *testing.T) { + testCases := []struct { + name string + withoutLockfile bool + expected string + }{ + {name: "default mode", withoutLockfile: false, expected: filepath.Join(testProjectDir, "legacy-locks")}, + {name: "lock-file-free mode", withoutLockfile: true, expected: ""}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + ctx := newTestCtxWithXDGConfigHome() + writeProjectConfig(t, ctx, ` +[project] +lock-dir = "legacy-locks" +`) + + _, config, err := projectconfig.LoadProjectConfig( + ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, + false, testCase.withoutLockfile, + ) + require.NoError(t, err) + require.NotNil(t, config) + assert.Equal(t, testCase.expected, config.Project.LockDir) + }) + } +} diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index 4c2a38ce8..899ed8130 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -108,14 +108,41 @@ func (f ConfigFile) Validate() error { return err } - // Per-component snapshot timestamps are not allowed. Components inherit - // the snapshot from the distro/group default-component-config or the - // project's default-distro. Per-component snapshots would create - // non-deterministic builds that the lock file cannot reliably track. - // Use an explicit 'upstream-commit' pin instead. - - // Validate overlay configurations for each component. - for componentName, component := range f.Components { + if err := validateComponentConfigs(f.Components); err != nil { + return err + } + + if err := validateTestSuites(f.TestSuites); err != nil { + return err + } + + if err := validateTestDefinitions(f.Tests); err != nil { + return err + } + + return nil +} + +// validateNonComponentFields validates every field except the component +// definitions. Lock-file-free mode merges component definitions across config +// files with override semantics, so a single file may legitimately be +// incomplete; components are validated once the whole project is assembled. +func (f ConfigFile) validateNonComponentFields() error { + f.Components = nil + + return f.Validate() +} + +// validateComponentConfigs validates the parts of a component definition that +// the struct validator cannot express. +// +// Per-component snapshot timestamps are not allowed. Components inherit the +// snapshot from the distro/group default-component-config or the project's +// default-distro. Per-component snapshots would create non-deterministic builds +// that the lock file cannot reliably track. Use an explicit 'upstream-commit' +// pin instead. +func validateComponentConfigs(components map[string]ComponentConfig) error { + for componentName, component := range components { for i, overlay := range component.Overlays { err := overlay.Validate() if err != nil { @@ -142,14 +169,6 @@ func (f ConfigFile) Validate() error { } } - if err := validateTestSuites(f.TestSuites); err != nil { - return err - } - - if err := validateTestDefinitions(f.Tests); err != nil { - return err - } - return nil } @@ -652,3 +671,19 @@ func (f ConfigFile) Serialize(fs opctx.FS, filePath string) error { return nil } + +// validateComponentStructs runs the struct validator over each component +// definition. Config files declare components with a 'dive' validation tag, so +// this is only needed when component validation is deferred until the whole +// project has been merged (lock-file-free mode). +func validateComponentStructs(components map[string]ComponentConfig) error { + validate := validator.New() + + for componentName, component := range components { + if err := validate.Struct(&component); err != nil { + return fmt.Errorf("invalid component %#q:\n%w", componentName, err) + } + } + + return nil +} diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index 2891dfd51..6ed0406d8 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -29,11 +29,23 @@ var ( ErrCircularInclude = errors.New("circular include detected") ) +// loadOptions carries the load-time modes that change how configuration files are +// parsed, merged, and validated. +type loadOptions struct { + // permissiveConfigParsing ignores unknown fields and downgrades validation + // failures to warnings. + permissiveConfigParsing bool + + // withoutLockfile selects lock-file-free mode, where component definitions are + // merged with override semantics and validated only once fully assembled. + withoutLockfile bool +} + // Loads and resolves the project configuration files located at the given path. Referenced include files // are recursively loaded and appropriately merged. If multiple file paths are provided, they are each // fully loaded and merged in specified order, with later files overriding earlier ones. func loadAndResolveProjectConfig( - fs opctx.FS, permissiveConfigParsing bool, configFilePaths ...string, + fs opctx.FS, options loadOptions, configFilePaths ...string, ) (*ProjectConfig, error) { resolvedCfg := &ProjectConfig{ ComponentGroups: make(map[string]ComponentGroupConfig), @@ -49,16 +61,16 @@ func loadAndResolveProjectConfig( for _, configFilePath := range configFilePaths { // Load the project config file and all transitive includes. - err := loadAndMergeConfigWithIncludes(resolvedCfg, fs, configFilePath, permissiveConfigParsing) + err := loadAndMergeConfigWithIncludes(resolvedCfg, fs, configFilePath, options) if err != nil { return nil, err } } // Validate the resulting configuration. - err := resolvedCfg.Validate() + err := resolvedCfg.validate(options.withoutLockfile) if err != nil { - if permissiveConfigParsing { + if options.permissiveConfigParsing { slog.Warn( "Project config validation failed; continuing due to '--permissive-config'", "configFiles", configFilePaths, @@ -74,16 +86,16 @@ func loadAndResolveProjectConfig( func loadAndMergeConfigWithIncludes( configToUpdate *ProjectConfig, fs opctx.FS, filePath string, - permissiveConfigParsing bool, + options loadOptions, ) error { // Load the project config file and all transitive includes. - loadedCfgs, err := loadProjectConfigWithIncludes(fs, filePath, permissiveConfigParsing, nil) + loadedCfgs, err := loadProjectConfigWithIncludes(fs, filePath, options, nil) if err != nil { return err } // Go through all the loaded configs and merge them into the resolved config. - err = mergeConfigFiles(configToUpdate, loadedCfgs) + err = mergeConfigFiles(configToUpdate, loadedCfgs, options) if err != nil { return err } @@ -91,9 +103,9 @@ func loadAndMergeConfigWithIncludes( return nil } -func mergeConfigFiles(resolvedCfg *ProjectConfig, loadedCfgs []*ConfigFile) error { +func mergeConfigFiles(resolvedCfg *ProjectConfig, loadedCfgs []*ConfigFile, options loadOptions) error { for _, loadedCfg := range loadedCfgs { - err := mergeConfigFile(resolvedCfg, loadedCfg) + err := mergeConfigFile(resolvedCfg, loadedCfg, options) if err != nil { return err } @@ -102,7 +114,7 @@ func mergeConfigFiles(resolvedCfg *ProjectConfig, loadedCfgs []*ConfigFile) erro return nil } -func mergeConfigFile(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { +func mergeConfigFile(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile, options loadOptions) error { if loadedCfg.Project != nil { err := resolvedCfg.Project.MergeUpdatesFrom(loadedCfg.Project.WithAbsolutePaths(loadedCfg.dir)) if err != nil { @@ -118,7 +130,7 @@ func mergeConfigFile(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { return err } - if err := mergeComponents(resolvedCfg, loadedCfg); err != nil { + if err := mergeComponents(resolvedCfg, loadedCfg, options); err != nil { return err } @@ -221,10 +233,20 @@ func mergeComponentGroups(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) err return nil } -// mergeComponents merges component definitions from a loaded config file into -// the resolved config. Components support additive merging: if a component +// mergeComponents merges component definitions from a loaded config file into the +// resolved config, using the merge semantics selected by the load mode. +func mergeComponents(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile, options loadOptions) error { + if options.withoutLockfile { + return mergeComponentsWithOverride(resolvedCfg, loadedCfg) + } + + return mergeComponentsAdditively(resolvedCfg, loadedCfg) +} + +// mergeComponentsAdditively merges component definitions from a loaded config file +// into the resolved config. Components support additive merging: if a component // already exists, its fields are updated from the new definition. -func mergeComponents(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { +func mergeComponentsAdditively(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { for componentName, component := range loadedCfg.Components { // Fill out fields not explicitly serialized. component.Name = componentName @@ -252,6 +274,38 @@ func mergeComponents(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { return nil } +// mergeComponentsWithOverride merges component definitions from a loaded config +// file into the resolved config so that later definitions override fields from +// earlier files. Lock-file-free mode relies on this so that a generated +// upstream-commit config can replace a component's configured commit pin. +func mergeComponentsWithOverride(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { + for componentName, component := range loadedCfg.Components { + // Fill out fields not explicitly serialized. + component.Name = componentName + component.SourceConfigFile = loadedCfg + + // Track commit provenance separately from the component's primary TOML. + // Synthetic history must follow the file that actually changed the pin, + // even when another partial component definition is merged later. + if component.Spec.UpstreamCommit != "" { + component.upstreamCommitConfigFile = loadedCfg + } + + resolvedComponent := component.WithAbsolutePaths(loadedCfg.dir) + if existing, ok := resolvedCfg.Components[componentName]; ok { + if err := existing.MergeOverridesFrom(resolvedComponent); err != nil { + return fmt.Errorf("failed to merge component %#q:\n%w", componentName, err) + } + + resolvedCfg.Components[componentName] = existing + } else { + resolvedCfg.Components[componentName] = *resolvedComponent + } + } + + return nil +} + // mergeImages merges image definitions from a loaded config file into the // resolved config. Duplicate image names are not allowed. func mergeImages(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { @@ -369,7 +423,7 @@ func mergeTestGroups(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { } func loadProjectConfigWithIncludes( - fs opctx.FS, filePath string, permissiveConfigParsing bool, + fs opctx.FS, filePath string, options loadOptions, seen map[string]bool, ) ([]*ConfigFile, error) { absFilePath, err := filepath.Abs(filePath) @@ -391,7 +445,7 @@ func loadProjectConfigWithIncludes( seen[absFilePath] = true // Load the immediate config file. - cfg, err := loadProjectConfigFile(fs, filePath, permissiveConfigParsing) + cfg, err := loadProjectConfigFile(fs, filePath, options) if err != nil { return nil, err } @@ -424,7 +478,7 @@ func loadProjectConfigWithIncludes( absIncludePath := makeAbsolute(cfg.dir, includePath) includeCfgs, err := loadProjectConfigWithIncludes( - fs, absIncludePath, permissiveConfigParsing, seen, + fs, absIncludePath, options, seen, ) if err != nil { return nil, err @@ -438,7 +492,7 @@ func loadProjectConfigWithIncludes( } func loadProjectConfigFile( - fs opctx.FS, filePath string, permissiveConfigParsing bool, + fs opctx.FS, filePath string, options loadOptions, ) (*ConfigFile, error) { slog.Debug("Loading project config", "filePath", filePath) @@ -454,7 +508,7 @@ func loadProjectConfigFile( decoder := toml.NewDecoder(projectFile) - if !permissiveConfigParsing { + if !options.permissiveConfigParsing { decoder.DisallowUnknownFields() } @@ -482,8 +536,16 @@ func loadProjectConfigFile( cfg.sourcePath = absFilePath cfg.dir = filepath.Dir(absFilePath) - // Make sure that the read data is internally consistent. - err = cfg.Validate() + // Make sure that the read data is internally consistent. In lock-file-free + // mode, component definitions support override merging across files, so they + // are validated only after the complete project configuration is assembled. + // Syntax and unknown fields were already checked by the TOML decoder. + if options.withoutLockfile { + err = cfg.validateNonComponentFields() + } else { + err = cfg.Validate() + } + if err != nil { return nil, err } diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index 50a53fe68..56cc5dadd 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -22,7 +22,7 @@ var testConfigPath = filepath.Join("/project", DefaultConfigFileName) func TestLoadAndResolveProjectConfig(t *testing.T) { ctx := testctx.NewCtx() - config, err := loadAndResolveProjectConfig(ctx.FS(), false, "/non/existent") + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, "/non/existent") require.ErrorIs(t, err, os.ErrNotExist) assert.Nil(t, config) } @@ -31,7 +31,7 @@ func TestLoadAndResolveProjectConfig_SyntaxError(t *testing.T) { ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte("///"), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Nil(t, config) } @@ -41,7 +41,7 @@ func TestLoadAndResolveProjectConfig_BadSchema(t *testing.T) { require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte("[non-existent-section]"), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Nil(t, config) } @@ -52,7 +52,7 @@ func TestLoadAndResolveProjectConfig_BadSchema_PermissiveParsing(t *testing.T) { []byte("[non-existent-section]"), fileperms.PrivateFile)) // With permissive parsing enabled, unknown fields should be silently ignored. - config, err := loadAndResolveProjectConfig(ctx.FS(), true, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testConfigPath) require.NoError(t, err) assert.NotNil(t, config) } @@ -70,12 +70,12 @@ key = "value" require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) // Strict parsing should fail on the unknown section. - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Nil(t, config) // Permissive parsing should succeed and preserve the known fields. - config, err = loadAndResolveProjectConfig(ctx.FS(), true, testConfigPath) + config, err = loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testConfigPath) require.NoError(t, err) require.NotNil(t, config) assert.Equal(t, "my project", config.Project.Description) @@ -110,12 +110,12 @@ key = "value" } // Strict parsing should fail because the included file has an unknown section. - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.Error(t, err) assert.Nil(t, config) // Permissive parsing should succeed and resolve fields from both files. - config, err = loadAndResolveProjectConfig(ctx.FS(), true, testFiles[0].path) + config, err = loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testFiles[0].path) require.NoError(t, err) require.NotNil(t, config) assert.Equal(t, "my project", config.Project.Description) @@ -154,7 +154,7 @@ ref = "0123456789abcdef0123456789abcdef01234567" require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) } - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) require.Contains(t, config.Components, "bash") require.Contains(t, config.Tests, "bash-fedora-shell") @@ -173,12 +173,12 @@ components = ["missing-component"] require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) // Strict parsing should fail because the referenced component is undefined. - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.ErrorIs(t, err, ErrUndefinedComponent) assert.Nil(t, config) // Permissive parsing should ignore the validation error and return the config. - config, err = loadAndResolveProjectConfig(ctx.FS(), true, testConfigPath) + config, err = loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testConfigPath) require.NoError(t, err) require.NotNil(t, config) assert.Contains(t, config.ComponentGroups, "my-group") @@ -188,7 +188,7 @@ func TestLoadAndResolveProjectConfig_EmptyFile(t *testing.T) { ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte{}, fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) // Check config @@ -213,7 +213,7 @@ specs = ["SPECS/**/*.spec"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) // Confirm parsed data. @@ -234,7 +234,7 @@ func TestLoadAndResolveProjectConfig_Component(t *testing.T) { ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) // Confirm parsed data. @@ -266,7 +266,7 @@ dist-git-branch = "NinePointThree" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.Distros, "abc") { @@ -301,7 +301,7 @@ output-dir = "out" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) // Validate config, making sure paths were made absolute. @@ -342,7 +342,7 @@ log-dir = "artifacts/logs" require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) } - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.NoError(t, err) // Validate resolved config. @@ -379,7 +379,7 @@ upstream-commit = "bbb2222" require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) } - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.NoError(t, err) // The included file is loaded after the parent, so its values override. @@ -409,7 +409,7 @@ includes = ["include.toml"] require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) } - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.ErrorIs(t, err, ErrDuplicateComponentGroups) } @@ -444,7 +444,7 @@ dist-git-branch = "TenPointZero" require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath1, []byte(configContents1), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath2, []byte(configContents2), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, configPath1, configPath2) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, configPath1, configPath2) require.NoError(t, err) if assert.Contains(t, config.Distros, "abc") { @@ -501,7 +501,7 @@ upstream-commit = "bbb2222" require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath1, []byte(configContents1), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath2, []byte(configContents2), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, configPath1, configPath2) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, configPath1, configPath2) require.NoError(t, err) // The second file's upstream-commit should override the first. @@ -538,7 +538,7 @@ upstream-commit = "def5678" require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath1, []byte(configContents1), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath2, []byte(configContents2), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, configPath1, configPath2) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, configPath1, configPath2) require.NoError(t, err) comp := config.Components["curl"] @@ -580,7 +580,7 @@ upstream-commit = "bbb2222" require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) } - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.NoError(t, err) comp := config.Components["abc"] @@ -623,7 +623,7 @@ type = "upstream" require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath1, []byte(configContents1), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath2, []byte(configContents2), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, configPath1, configPath2) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, configPath1, configPath2) require.NoError(t, err) // All three components should be present. @@ -669,7 +669,7 @@ value = "libcurl" require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath1, []byte(configContents1), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath2, []byte(configContents2), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, configPath1, configPath2) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, configPath1, configPath2) require.NoError(t, err) comp := config.Components["pkg"] @@ -696,7 +696,7 @@ specs = ["SPECS/**/*.spec"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) // Confirm group parsed correctly. @@ -729,7 +729,7 @@ upstream-status = "upstreamable" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.ComponentGroups, "core") { @@ -754,7 +754,7 @@ bugs = [{ url = "https://example.com/bug/1" }] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Nil(t, config) } @@ -786,7 +786,7 @@ components = ["shared", "only-beta"] require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) } - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.NoError(t, err) // "shared" belongs to both groups. @@ -814,7 +814,7 @@ without = ["docs"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.ComponentGroups, "core") { @@ -836,7 +836,7 @@ specs = ["SPECS/**/*.spec"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) // No members means no GroupsByComponent entries. @@ -851,7 +851,7 @@ includes = ["include.toml"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.ErrorIs(t, err, os.ErrNotExist) assert.Nil(t, config) } @@ -864,7 +864,7 @@ includes = ["*non-existent*.toml"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) } @@ -877,7 +877,7 @@ rpm-channel = "base" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) assert.Equal(t, "base", config.DefaultPackageConfig.Publish.RPMChannel) @@ -907,7 +907,7 @@ rpm-channel = "stable" require.NoError(t, fileutils.WriteFile(ctx.FS(), f.path, []byte(f.contents), fileperms.PrivateFile)) } - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.NoError(t, err) // The later-loaded file wins. @@ -934,7 +934,7 @@ rpm-channel = "second" require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath1, []byte(configContents1), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath2, []byte(configContents2), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, configPath1, configPath2) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, configPath1, configPath2) require.NoError(t, err) assert.Equal(t, "second", config.DefaultPackageConfig.Publish.RPMChannel) @@ -960,7 +960,7 @@ rpm-channel = "none" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) require.Len(t, config.PackageGroups, 2) @@ -1003,7 +1003,7 @@ packages = ["wget2-devel"] require.NoError(t, fileutils.WriteFile(ctx.FS(), f.path, []byte(f.contents), fileperms.PrivateFile)) } - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.ErrorIs(t, err, ErrDuplicatePackageGroups) } @@ -1026,7 +1026,7 @@ packages = ["wget2-devel"] require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath1, []byte(configContents1), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(ctx.FS(), configPath2, []byte(configContents2), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, configPath1, configPath2) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, configPath1, configPath2) require.ErrorIs(t, err, ErrDuplicatePackageGroups) } @@ -1039,7 +1039,7 @@ packages = ["curl-devel", ""] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Contains(t, err.Error(), "packages[1]") assert.Contains(t, err.Error(), "must not be empty") @@ -1054,7 +1054,7 @@ packages = ["curl-devel", "wget2-devel", "curl-devel"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Contains(t, err.Error(), "curl-devel") assert.Contains(t, err.Error(), "more than once") @@ -1072,7 +1072,7 @@ packages = ["wget2-devel", "bash-devel"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Contains(t, err.Error(), "wget2-devel") assert.Contains(t, err.Error(), "may only belong to one group") @@ -1089,7 +1089,7 @@ rpm-channel = "devel" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.Components, "curl") { @@ -1130,7 +1130,7 @@ ref = "abcdef0123456789abcdef0123456789abcdef01" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) require.Len(t, config.TestSuites, 2) @@ -1193,7 +1193,7 @@ test-paths = ["other/"] require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) } - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testFiles[0].path) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testFiles[0].path) require.ErrorIs(t, err, ErrDuplicateTestSuites) } @@ -1206,7 +1206,7 @@ type = "unsupported" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.ErrorIs(t, err, ErrUnknownTestType) } @@ -1221,7 +1221,7 @@ type = "pytest" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.ErrorIs(t, err, ErrMissingTestField) } @@ -1236,7 +1236,7 @@ description = "no type set" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) require.ErrorIs(t, err, ErrMissingTestField) assert.Contains(t, err.Error(), "type") @@ -1256,7 +1256,7 @@ working-dir = "tests" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Contains(t, err.Error(), "invalid test suite name") } @@ -1276,7 +1276,7 @@ test-paths = ["test_smoke.py"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Contains(t, err.Error(), "invalid test name") } @@ -1300,7 +1300,7 @@ test-suites = [{ name = "smoke" }] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.Images, "myimage") { @@ -1321,7 +1321,7 @@ test-suites = [{ name = "nonexistent" }] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) require.ErrorIs(t, err, ErrUndefinedTestSuite) assert.Contains(t, err.Error(), "nonexistent") @@ -1341,7 +1341,7 @@ cvm = true ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.Images, "myimage") { @@ -1367,7 +1367,7 @@ name = "smoke_test" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.Tests, "smoke-test") { @@ -1387,7 +1387,7 @@ channel = "rpm-base" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), true, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testConfigPath) require.NoError(t, err) // The deprecated field is preserved; no migration happens at load time. @@ -1412,7 +1412,7 @@ channel = "rpm-sdk" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), true, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testConfigPath) require.NoError(t, err) require.Contains(t, config.PackageGroups, "my-group") @@ -1437,7 +1437,7 @@ channel = "devel" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), true, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testConfigPath) require.NoError(t, err) require.Contains(t, config.Components, "curl") @@ -1462,7 +1462,7 @@ rpm-channel = "new-channel" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), true, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{permissiveConfigParsing: true}, testConfigPath) require.NoError(t, err) // Both fields are preserved as loaded. @@ -1487,7 +1487,7 @@ test-paths = ["cases/"] ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) if assert.Contains(t, config.TestSuites, "smoke") { @@ -1510,7 +1510,7 @@ install = "invalid" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.ErrorIs(t, err, ErrInvalidInstallMode) } @@ -1521,7 +1521,7 @@ func TestLoadAndResolveProjectConfig_CircularInclude(t *testing.T) { require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(`includes = ["azldev.toml"]`), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.ErrorIs(t, err, ErrCircularInclude) }) @@ -1535,7 +1535,7 @@ func TestLoadAndResolveProjectConfig_CircularInclude(t *testing.T) { require.NoError(t, fileutils.WriteFile(ctx.FS(), includePath, []byte(`includes = ["azldev.toml"]`), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.ErrorIs(t, err, ErrCircularInclude) }) @@ -1554,7 +1554,7 @@ func TestLoadAndResolveProjectConfig_CircularInclude(t *testing.T) { require.NoError(t, fileutils.WriteFile(ctx.FS(), cPath, []byte(`includes = ["azldev.toml"]`), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.ErrorIs(t, err, ErrCircularInclude) }) } @@ -1570,7 +1570,7 @@ expected = true ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - _, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.Error(t, err) assert.Contains(t, err.Error(), "expected-reason") }) @@ -1586,8 +1586,152 @@ expected-reason = "Known upstream issue #456" ctx := testctx.NewCtx() require.NoError(t, fileutils.WriteFile(ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile)) - config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) require.NoError(t, err) assert.True(t, config.Components["test-pkg"].Build.Failure.Expected) }) } + +func TestLoadAndResolveProjectConfig_WithoutLockfile_MergesComponents(t *testing.T) { + testFiles := []struct { + path string + contents string + }{ + {testConfigPath, ` +includes = ["include.toml", "later.toml"] + +[components.abc] +spec = { type = "upstream", upstream-distro = { name = "fedora", version = "rawhide" }, upstream-name = "source-abc" } +build = { defines = { with_feature = "1" } } +`}, + {"/project/include.toml", ` +[components.abc] +spec = { upstream-commit = "abcdef1234567" } +`}, + {"/project/later.toml", ` +[components.abc.release] +calculation = "static" +`}, + } + + ctx := testctx.NewCtx() + for _, testFile := range testFiles { + require.NoError(t, fileutils.WriteFile( + ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile, + )) + } + + config, err := loadAndResolveProjectConfig( + ctx.FS(), loadOptions{withoutLockfile: true}, testFiles[0].path, + ) + require.NoError(t, err) + require.Contains(t, config.Components, "abc") + + component := config.Components["abc"] + assert.Equal(t, SpecSourceTypeUpstream, component.Spec.SourceType) + assert.Equal(t, DistroReference{Name: "fedora", Version: "rawhide"}, component.Spec.UpstreamDistro) + assert.Equal(t, "source-abc", component.Spec.UpstreamName) + assert.Equal(t, "abcdef1234567", component.Spec.UpstreamCommit) + assert.Equal(t, ReleaseCalculationStatic, component.Release.Calculation) + assert.Equal(t, map[string]string{"with_feature": "1"}, component.Build.Defines) + require.NotNil(t, component.SourceConfigFile) + assert.Equal(t, "/project/later.toml", component.SourceConfigFile.sourcePath) + require.NotNil(t, component.UpstreamCommitConfigFile()) + assert.Equal(t, "/project/include.toml", component.UpstreamCommitConfigFile().sourcePath) +} + +func TestLoadAndResolveProjectConfig_WithoutLockfile_ValidatesComponentsAfterMerge(t *testing.T) { + testFiles := []struct { + path string + contents string + }{ + {testConfigPath, ` +includes = ["pin.toml", "component.toml"] +`}, + {"/project/pin.toml", ` +[components.abc.spec] +upstream-commit = "abcdef1234567" +`}, + {"/project/component.toml", ` +[components.abc.spec] +type = "upstream" +upstream-distro = { name = "fedora", version = "rawhide" } +`}, + } + + ctx := testctx.NewCtx() + for _, testFile := range testFiles { + require.NoError(t, fileutils.WriteFile( + ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile, + )) + } + + config, err := loadAndResolveProjectConfig( + ctx.FS(), loadOptions{withoutLockfile: true}, testConfigPath, + ) + require.NoError(t, err) + require.Contains(t, config.Components, "abc") + assert.Equal(t, SpecSourceTypeUpstream, config.Components["abc"].Spec.SourceType) + assert.Equal(t, "abcdef1234567", config.Components["abc"].Spec.UpstreamCommit) + + // The default lock-file mode validates each config file on its own, so the + // partial definition in pin.toml is rejected there. + _, err = loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, testConfigPath) + require.Error(t, err) +} + +func TestLoadAndResolveProjectConfig_WithoutLockfile_RejectsInvalidComponentAfterMerge(t *testing.T) { + const configContents = ` +[components.abc.spec] +upstream-commit = "abcdef1234567" +` + + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile( + ctx.FS(), testConfigPath, []byte(configContents), fileperms.PrivateFile, + )) + + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{withoutLockfile: true}, testConfigPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "UpstreamCommit") +} + +func TestLoadAndResolveProjectConfig_WithoutLockfile_ComponentOverridesEarlierGeneratedCommit(t *testing.T) { + testFiles := []struct { + path string + contents string + }{ + {testConfigPath, ` +includes = ["generated.toml", "component.toml"] +`}, + {"/project/generated.toml", ` +[components.abc.spec] +upstream-commit = "abcdef1234567" +`}, + {"/project/component.toml", ` +[components.abc.spec] +type = "upstream" +upstream-commit = "1234567abcdef" +upstream-distro = { name = "fedora", version = "rawhide" } +`}, + } + + ctx := testctx.NewCtx() + for _, testFile := range testFiles { + require.NoError(t, fileutils.WriteFile( + ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile, + )) + } + + config, err := loadAndResolveProjectConfig( + ctx.FS(), loadOptions{withoutLockfile: true}, testConfigPath, + ) + require.NoError(t, err) + require.Contains(t, config.Components, "abc") + + component := config.Components["abc"] + assert.Equal(t, SpecSourceTypeUpstream, component.Spec.SourceType) + assert.Equal(t, "1234567abcdef", component.Spec.UpstreamCommit) + require.NotNil(t, component.UpstreamCommitConfigFile()) + assert.Equal(t, "/project/component.toml", component.UpstreamCommitConfigFile().sourcePath) +} diff --git a/internal/projectconfig/project.go b/internal/projectconfig/project.go index 1788e9f0a..4b331e2bb 100644 --- a/internal/projectconfig/project.go +++ b/internal/projectconfig/project.go @@ -80,11 +80,28 @@ func NewProjectConfig() ProjectConfig { // Validates the configuration, returning an error if any semantic errors are found. func (cfg *ProjectConfig) Validate() error { + return cfg.validate(false) +} + +// validate checks the assembled project configuration. In lock-file-free mode the +// component definitions are validated here rather than per config file, because +// override merging lets a single file hold a partial definition. +func (cfg *ProjectConfig) validate(withoutLockfile bool) error { err := validator.New().Struct(cfg) if err != nil { return fmt.Errorf("config error:\n%w", err) } + if withoutLockfile { + if err := validateComponentStructs(cfg.Components); err != nil { + return err + } + + if err := validateComponentConfigs(cfg.Components); err != nil { + return err + } + } + if err := validateComponentGroupMembership(cfg.ComponentGroups, cfg.Components); err != nil { return err } diff --git a/internal/projectgen/projectgen_test.go b/internal/projectgen/projectgen_test.go index e1200e5e3..f111ab048 100644 --- a/internal/projectgen/projectgen_test.go +++ b/internal/projectgen/projectgen_test.go @@ -36,6 +36,7 @@ func requireProjectHasValidDefaultConfig(t *testing.T, ctx opctx.Ctx, projectPat t.TempDir(), nil, false, + false, ) require.NoError(t, err) @@ -66,7 +67,8 @@ default-distro = { name = "other", version = "42.42" } // Load the project. foundProjectDir, config, err := projectconfig.LoadProjectConfig( - ctx.FS(), ctx.OSEnv(), testProjectPath, false /*disable default config?*/, t.TempDir(), nil, false, + ctx.FS(), ctx.OSEnv(), testProjectPath, false, /*disable default config?*/ + t.TempDir(), nil, false, false, ) require.NoError(t, err) diff --git a/internal/upstreamcommit/store.go b/internal/upstreamcommit/store.go new file mode 100644 index 000000000..434900290 --- /dev/null +++ b/internal/upstreamcommit/store.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package upstreamcommit manages generated component configuration files that +// pin resolved upstream commits. +package upstreamcommit + +import ( + "errors" + "fmt" + "log/slog" + "path/filepath" + "sort" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + toml "github.com/pelletier/go-toml/v2" +) + +const fileExtension = ".toml" + +// DefaultDir is the project-relative directory that holds the generated +// per-component upstream-commit TOML files. +const DefaultDir = "base/upstream-commits" + +const generatedFileHeader = `# This file was generated by 'azldev component refresh-upstream-commit' +# Do not edit this file, changes will be lost +# For more details see 'azldev component refresh-upstream-commit --help' +` + +type generatedConfig struct { + Components map[string]generatedComponent `toml:"components"` +} + +type generatedComponent struct { + Spec generatedSpec `toml:"spec"` +} + +type generatedSpec struct { + UpstreamCommit string `toml:"upstream-commit"` +} + +// Store reads and writes generated upstream-commit component configuration. +type Store struct { + fs opctx.FS + dir string +} + +// NewStore creates a store rooted at dir. +func NewStore(fs opctx.FS, dir string) *Store { + return &Store{fs: fs, dir: dir} +} + +// Dir returns the generated configuration directory. +func (s *Store) Dir() string { + return s.dir +} + +// Path returns the generated TOML path for componentName. +func (s *Store) Path(componentName string) (string, error) { + if err := fileutils.ValidateFilename(componentName); err != nil { + return "", fmt.Errorf("validating component name %#q for upstream commit TOML path:\n%w", + componentName, err) + } + + return filepath.Join(s.dir, componentName+fileExtension), nil +} + +// Get returns the upstream commit in a component's generated TOML file. +func (s *Store) Get(componentName string) (commit string, exists bool, err error) { + path, err := s.Path(componentName) + if err != nil { + return "", false, err + } + + exists, err = fileutils.Exists(s.fs, path) + if err != nil { + return "", false, fmt.Errorf("checking upstream commit TOML %#q:\n%w", path, err) + } + + if !exists { + return "", false, nil + } + + data, err := fileutils.ReadFile(s.fs, path) + if err != nil { + return "", true, fmt.Errorf("reading upstream commit TOML %#q:\n%w", path, err) + } + + var config projectconfig.ConfigFile + if err := toml.Unmarshal(data, &config); err != nil { + return "", true, fmt.Errorf("parsing upstream commit TOML %#q:\n%w", path, err) + } + + component, ok := config.Components[componentName] + if !ok { + return "", true, fmt.Errorf( + "upstream commit TOML %#q does not define component %#q", path, componentName) + } + + return component.Spec.UpstreamCommit, true, nil +} + +// Exists reports whether a generated TOML exists for componentName. +func (s *Store) Exists(componentName string) (bool, error) { + path, err := s.Path(componentName) + if err != nil { + return false, err + } + + exists, err := fileutils.Exists(s.fs, path) + if err != nil { + return false, fmt.Errorf("checking upstream commit TOML %#q:\n%w", path, err) + } + + return exists, nil +} + +// Remove deletes the generated TOML for componentName if it exists. +func (s *Store) Remove(componentName string) (bool, error) { + path, err := s.Path(componentName) + if err != nil { + return false, err + } + + exists, err := s.Exists(componentName) + if err != nil { + return false, err + } + + if !exists { + return false, nil + } + + if err := s.fs.Remove(path); err != nil { + return false, fmt.Errorf("removing upstream commit TOML %#q:\n%w", path, err) + } + + return true, nil +} + +// Save writes a generated component TOML override containing only upstreamCommit. +func (s *Store) Save(componentName, upstreamCommit string) error { + path, err := s.Path(componentName) + if err != nil { + return err + } + + config := generatedConfig{ + Components: map[string]generatedComponent{ + componentName: { + Spec: generatedSpec{UpstreamCommit: upstreamCommit}, + }, + }, + } + + data, err := toml.Marshal(config) + if err != nil { + return fmt.Errorf("serializing upstream commit TOML %#q:\n%w", path, err) + } + + data = append([]byte(generatedFileHeader), data...) + + if err := fileutils.MkdirAll(s.fs, s.dir); err != nil { + return fmt.Errorf("creating upstream commit TOML directory %#q:\n%w", s.dir, err) + } + + if err := fileutils.WriteFile(s.fs, path, data, fileperms.PublicFile); err != nil { + return fmt.Errorf("writing upstream commit TOML %#q:\n%w", path, err) + } + + return nil +} + +// FindOrphans returns generated component TOMLs that do not correspond to an +// upstream component in components. +func (s *Store) FindOrphans( + components map[string]projectconfig.ComponentConfig, +) ([]string, error) { + entries, err := fileutils.ReadDir(s.fs, s.dir) + if err != nil { + exists, existsErr := fileutils.DirExists(s.fs, s.dir) + if existsErr != nil { + return nil, fmt.Errorf("checking upstream commit TOML directory %#q:\n%w", s.dir, existsErr) + } + + if !exists { + return nil, nil + } + + return nil, fmt.Errorf("reading upstream commit TOML directory %#q:\n%w", s.dir, err) + } + + var orphans []string + + for _, entry := range entries { + if entry.IsDir() || strings.HasPrefix(entry.Name(), ".") || + !strings.HasSuffix(entry.Name(), fileExtension) { + continue + } + + name := strings.TrimSuffix(entry.Name(), fileExtension) + + component, ok := components[name] + if !ok || component.Spec.SourceType != projectconfig.SpecSourceTypeUpstream { + orphans = append(orphans, name) + } + } + + sort.Strings(orphans) + + return orphans, nil +} + +// PruneOrphans removes generated TOMLs that are no longer needed. +func (s *Store) PruneOrphans(components map[string]projectconfig.ComponentConfig) (int, error) { + orphans, err := s.FindOrphans(components) + if err != nil { + return 0, err + } + + var errs []error + + pruned := 0 + + for _, name := range orphans { + path, pathErr := s.Path(name) + if pathErr != nil { + errs = append(errs, pathErr) + + continue + } + + slog.Info("Removing orphan upstream commit TOML", "component", name) + + if removeErr := s.fs.Remove(path); removeErr != nil { + errs = append(errs, fmt.Errorf("removing upstream commit TOML for %#q:\n%w", name, removeErr)) + + continue + } + + pruned++ + } + + if len(errs) > 0 { + return pruned, errors.Join(errs...) + } + + return pruned, nil +} diff --git a/internal/upstreamcommit/store_test.go b/internal/upstreamcommit/store_test.go new file mode 100644 index 000000000..923d56bd1 --- /dev/null +++ b/internal/upstreamcommit/store_test.go @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package upstreamcommit_test + +import ( + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/upstreamcommit" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testStoreDir = "/project/base/upstream-commits" + +func TestStoreSaveAndGet(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + + require.NoError(t, store.Save("bash", "abcdef1234567")) + + data, err := fileutils.ReadFile(ctx.FS(), filepath.Join(testStoreDir, "bash.toml")) + require.NoError(t, err) + assert.Equal(t, `# This file was generated by 'azldev component refresh-upstream-commit' +# Do not edit this file, changes will be lost +# For more details see 'azldev component refresh-upstream-commit --help' +[components] +[components.bash] +[components.bash.spec] +upstream-commit = 'abcdef1234567' +`, string(data)) + + commit, exists, err := store.Get("bash") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "abcdef1234567", commit) + + require.NoError(t, store.Save("bash", "1234567abcdef")) + commit, exists, err = store.Get("bash") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "1234567abcdef", commit) +} + +func TestStoreSaveReplacesExistingGeneratedContent(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + path := filepath.Join(testStoreDir, "bash.toml") + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testStoreDir)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte(` +[components.bash.spec] +type = "local" +path = "bash.spec" +upstream-commit = "oldcommit" + +[components.bash.build] +defines = { stale = "value" } +`), fileperms.PublicFile)) + + require.NoError(t, store.Save("bash", "abcdef1234567")) + + data, err := fileutils.ReadFile(ctx.FS(), path) + require.NoError(t, err) + assert.NotContains(t, string(data), "type") + assert.NotContains(t, string(data), "path") + assert.NotContains(t, string(data), "defines") + assert.Contains(t, string(data), "upstream-commit = 'abcdef1234567'") +} + +func TestStoreGetMissing(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + + commit, exists, err := store.Get("missing") + require.NoError(t, err) + assert.False(t, exists) + assert.Empty(t, commit) +} + +func TestStoreExistsAndRemove(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + + exists, err := store.Exists("bash") + require.NoError(t, err) + assert.False(t, exists) + + require.NoError(t, store.Save("bash", "abcdef1234567")) + exists, err = store.Exists("bash") + require.NoError(t, err) + assert.True(t, exists) + + removed, err := store.Remove("bash") + require.NoError(t, err) + assert.True(t, removed) + + removed, err = store.Remove("bash") + require.NoError(t, err) + assert.False(t, removed) +} + +func TestStoreRejectsInvalidComponentName(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + + _, err := store.Path("../outside") + require.Error(t, err) + _, err = store.Exists("../outside") + require.Error(t, err) + _, err = store.Remove("../outside") + require.Error(t, err) + require.Error(t, store.Save("../outside", "abcdef1")) +} + +func TestStoreGetReportsInvalidTOMLAndSaveReplacesIt(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + path := filepath.Join(testStoreDir, "bash.toml") + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testStoreDir)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte("invalid = ["), fileperms.PublicFile)) + + _, exists, err := store.Get("bash") + assert.True(t, exists) + require.ErrorContains(t, err, "parsing upstream commit TOML") + + require.NoError(t, store.Save("bash", "abcdef1")) + commit, exists, err := store.Get("bash") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "abcdef1", commit) +} + +func TestStoreFindAndPruneOrphans(t *testing.T) { + ctx := testctx.NewCtx() + + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + for _, name := range []string{"kept", "local", "removed"} { + require.NoError(t, store.Save(name, "abcdef1")) + } + + components := map[string]projectconfig.ComponentConfig{ + "kept": { + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + }, + "local": { + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + }, + } + + orphans, err := store.FindOrphans(components) + require.NoError(t, err) + assert.Equal(t, []string{"local", "removed"}, orphans) + + pruned, err := store.PruneOrphans(components) + require.NoError(t, err) + assert.Equal(t, 2, pruned) + + _, exists, err := store.Get("kept") + require.NoError(t, err) + assert.True(t, exists) + _, exists, err = store.Get("removed") + require.NoError(t, err) + assert.False(t, exists) +} diff --git a/pkg/app/azldev_cli/azldev.go b/pkg/app/azldev_cli/azldev.go index 1f8782de7..786e5df6b 100644 --- a/pkg/app/azldev_cli/azldev.go +++ b/pkg/app/azldev_cli/azldev.go @@ -26,21 +26,35 @@ import ( // Main constructs the azldev CLI application, runs it with the process // arguments, and exits the process with the resulting status code. func Main() { - // Instantiate the main CLI app instance. - app := InstantiateApp() + args := os.Args[1:] + + // Instantiate the main CLI app instance. The arguments are needed up front + // because command registration depends on the global flags they carry. + app := InstantiateAppForArgs(args) // Execute! We'll get back an exit code that we will exit with. - ret := app.Execute(os.Args[1:]) + ret := app.Execute(args) os.Exit(ret) } // InstantiateApp constructs a new instance of the azldev CLI application with -// all subcommands registered. +// all subcommands registered for azldev's default (lock file) mode. func InstantiateApp() *azldev.App { + return InstantiateAppForArgs(nil) +} + +// InstantiateAppForArgs constructs a new instance of the azldev CLI application +// with the subcommands registered for the mode selected by args. Global flags +// that select a mode (such as '--without-lockfile') are hand-parsed from args +// before registration, because the registered command set differs by mode. +func InstantiateAppForArgs(args []string) *azldev.App { // Instantiate the main CLI application. app := azldev.NewApp(azldev.DefaultFileSystemFactory(), azldev.DefaultOSEnvFactory()) + // Resolve the global flags that command registration depends on. + app.PreParseGlobalFlags(args) + // Give top level command packages an opportunity to register their commands (or in some cases, // request post-init callbacks). advanced.OnAppInit(app) diff --git a/scenario/__snapshots__/TestMCPServerMode_1.snap.json b/scenario/__snapshots__/TestMCPServerMode_1.snap.json index 02459314b..20e599111 100755 --- a/scenario/__snapshots__/TestMCPServerMode_1.snap.json +++ b/scenario/__snapshots__/TestMCPServerMode_1.snap.json @@ -96,6 +96,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [ @@ -193,6 +198,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], @@ -288,6 +298,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], @@ -373,6 +388,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], @@ -445,6 +465,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], @@ -513,6 +538,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], @@ -586,6 +616,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], @@ -669,6 +704,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [ @@ -739,6 +779,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], @@ -826,6 +871,11 @@ "default": false, "description": "enable verbose output", "type": "boolean" + }, + "without-lockfile": { + "default": false, + "description": "preview: track resolved upstream commits in generated config instead of lock files", + "type": "boolean" } }, "required": [], diff --git a/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap index 827a71ea7..5d4be54da 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap @@ -30,6 +30,7 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap index 77ca08f7c..2ef8a077c 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap @@ -37,6 +37,7 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap index 77ca08f7c..2ef8a077c 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap @@ -37,6 +37,7 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap b/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap index 827a71ea7..5d4be54da 100755 --- a/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap @@ -30,6 +30,7 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap index 77ca08f7c..2ef8a077c 100755 --- a/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap @@ -37,6 +37,7 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap index 9711e319a..4c6577938 100755 --- a/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap @@ -37,6 +37,7 @@ Meta commands: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap index 77ca08f7c..2ef8a077c 100755 --- a/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap @@ -37,6 +37,7 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/without_lockfile_test.go b/scenario/without_lockfile_test.go new file mode 100644 index 000000000..23ccd4cbc --- /dev/null +++ b/scenario/without_lockfile_test.go @@ -0,0 +1,298 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build scenario + +package scenario_tests + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/cmdtest" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/projecttest" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/testhelpers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests cover the opt-in lock-file-free mode selected by the global +// '--without-lockfile' flag, and assert that omitting the flag keeps azldev's +// default lock-file behavior. + +// TestWithoutLockfile_GlobalFlagIsDocumented verifies that the preview flag is part +// of the CLI surface and defaults to off. +func TestWithoutLockfile_GlobalFlagIsDocumented(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + results, err := cmdtest.NewScenarioTest("--help").Locally().Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode) + assert.Contains(t, results.Stdout, "--without-lockfile") +} + +// TestWithoutLockfile_ComponentCommandsByMode verifies that the component command +// set matches the selected mode: the lock-file commands by default, and the +// refresh-upstream-commit command when the preview flag is passed. +func TestWithoutLockfile_ComponentCommandsByMode(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + testCases := []struct { + name string + args []string + expected []string + notExpected []string + }{ + { + name: "default mode", + args: []string{"component", "--help"}, + expected: []string{"update", "history", "query"}, + notExpected: []string{"refresh-upstream-commit"}, + }, + { + name: "explicitly disabled", + args: []string{"--without-lockfile=false", "component", "--help"}, + expected: []string{"update", "history", "query"}, + notExpected: []string{"refresh-upstream-commit"}, + }, + { + name: "lock-file-free mode", + args: []string{"--without-lockfile", "component", "--help"}, + expected: []string{"refresh-upstream-commit"}, + // The lock-file commands remain registered as hidden no-ops, so they + // must not be advertised in help. + notExpected: []string{"Refresh component lock files"}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + results, err := cmdtest.NewScenarioTest(testCase.args...).Locally().Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode) + + for _, expected := range testCase.expected { + assert.Contains(t, results.Stdout, expected) + } + + for _, notExpected := range testCase.notExpected { + assert.NotContains(t, results.Stdout, notExpected) + } + }) + } +} + +// TestWithoutLockfile_BuildFlagsByMode verifies that the lock-file-only component +// flags are registered only in the default mode. +func TestWithoutLockfile_BuildFlagsByMode(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + defaultResults, err := cmdtest.NewScenarioTest("component", "build", "--help").Locally().Run(t) + require.NoError(t, err) + require.Zero(t, defaultResults.ExitCode) + assert.Contains(t, defaultResults.Stdout, "--skip-lock-validation") + + previewResults, err := cmdtest.NewScenarioTest( + "--without-lockfile", "component", "build", "--help", + ).Locally().Run(t) + require.NoError(t, err) + require.Zero(t, previewResults.ExitCode) + assert.NotContains(t, previewResults.Stdout, "--skip-lock-validation") +} + +// TestWithoutLockfile_LegacyCommandIsNoOp verifies that a lock-file command invoked +// in lock-file-free mode reports that it does nothing instead of failing. +func TestWithoutLockfile_LegacyCommandIsNoOp(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + results, err := cmdtest.NewScenarioTest( + "--without-lockfile", "component", "update", "--all-components", + ).Locally().Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode) + assert.Contains(t, results.Stdout+results.Stderr, "no longer does anything") +} + +// TestWithoutLockfile_AgentSkillsByMode verifies that the emitted agent skills +// describe the workflow of the selected mode. +func TestWithoutLockfile_AgentSkillsByMode(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + defaultResults, err := cmdtest.NewScenarioTest("docs", "agent", "show").Locally().Run(t) + require.NoError(t, err) + require.Zero(t, defaultResults.ExitCode) + assert.Contains(t, defaultResults.Stdout, "azldev-update-component") + assert.NotContains(t, defaultResults.Stdout, "azldev-refresh-upstream-commit") + + previewResults, err := cmdtest.NewScenarioTest( + "--without-lockfile", "docs", "agent", "show", + ).Locally().Run(t) + require.NoError(t, err) + require.Zero(t, previewResults.ExitCode) + assert.Contains(t, previewResults.Stdout, "azldev-refresh-upstream-commit") + assert.NotContains(t, previewResults.Stdout, "azldev-update-component") +} + +// TestWithoutLockfile_ComponentChanged compares components across two commits in +// lock-file-free mode, where change detection compares the project configuration +// resolved at each ref instead of stored lock files. +// +// Flow: +// 1. Create a project with two local components (curl, bash). +// 2. Commit the project as the baseline. +// 3. In a second commit, change only curl's spec content. +// 4. Run 'azldev --without-lockfile component changed' between the two commits. +// 5. Assert curl is "changed" (its spec directory contents differ) and bash is +// "unchanged", with no lock files anywhere in the project. +func TestWithoutLockfile_ComponentChanged(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + azldevBin, err := testhelpers.FindTestBinary() + require.NoError(t, err) + + projectDir := t.TempDir() + + project := projecttest.NewDynamicTestProject( + projecttest.AddSpec(projecttest.NewSpec( + projecttest.WithName("curl"), + projecttest.WithVersion("8.0.0"), + projecttest.WithRelease("1%{?dist}"), + projecttest.WithBuildArch(projecttest.NoArch), + )), + projecttest.AddSpec(projecttest.NewSpec( + projecttest.WithName("bash"), + projecttest.WithVersion("5.2.0"), + projecttest.WithRelease("1%{?dist}"), + projecttest.WithBuildArch(projecttest.NoArch), + )), + projecttest.AddComponent(&projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: filepath.Join("specs", "curl", "curl.spec"), + }, + }), + projecttest.AddComponent(&projectconfig.ComponentConfig{ + Name: "bash", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: filepath.Join("specs", "bash", "bash.spec"), + }, + }), + projecttest.AddFile("distro.toml", minimalDistroTOML), + ) + + project.Serialize(t, projectDir) + patchProjectForLocal(t, projectDir) + + gitInDir(t, projectDir, "init") + gitInDir(t, projectDir, "config", "user.email", "test@test.com") + gitInDir(t, projectDir, "config", "user.name", "Test") + gitInDir(t, projectDir, "add", ".") + gitInDir(t, projectDir, "-c", "commit.gpgsign=false", "commit", "-m", "initial") + + fromRef := gitInDir(t, projectDir, "rev-parse", "HEAD") + + // Change only curl's spec content. + curlSpecPath := filepath.Join(projectDir, "specs", "curl", "curl.spec") + curlSpec, err := os.ReadFile(curlSpecPath) + require.NoError(t, err) + require.NoError(t, os.WriteFile( + curlSpecPath, + append(curlSpec, []byte("\n# changed by the scenario test\n")...), + fileperms.PublicFile, + )) + + gitInDir(t, projectDir, "add", filepath.Join("specs", "curl", "curl.spec")) + gitInDir(t, projectDir, "-c", "commit.gpgsign=false", "commit", "-m", "change curl") + + cmd := exec.CommandContext(t.Context(), + azldevBin, "--without-lockfile", "-C", projectDir, "--no-default-config", + "component", "changed", "--from", fromRef, "-a", "--include-unchanged", "-q", "-O", "json", + ) + + out, err := cmd.CombinedOutput() + require.NoError(t, err, "azldev failed: %s", string(out)) + + var results []changedResult + require.NoError(t, json.Unmarshal(out, &results), "failed to parse JSON: %s", string(out)) + + resultMap := make(map[string]changedResult, len(results)) + for _, result := range results { + resultMap[result.Component] = result + } + + curlResult, ok := resultMap["curl"] + require.True(t, ok, "curl should be in results") + assert.Equal(t, "changed", curlResult.ChangeType, "curl spec contents changed") + + bashResult, ok := resultMap["bash"] + require.True(t, ok, "bash should be in results (--all-components)") + assert.Equal(t, "unchanged", bashResult.ChangeType, "bash is untouched") + + // No lock files are consulted or created in this mode. + entries, err := os.ReadDir(projectDir) + require.NoError(t, err) + + for _, entry := range entries { + assert.NotEqual(t, "locks", entry.Name(), "lock-file-free mode must not create a lock directory") + assert.False(t, strings.HasSuffix(entry.Name(), ".lock")) + } +} + +// TestWithoutLockfile_MCPToolsByMode verifies that the MCP tool surface follows the +// selected mode's command set. +func TestWithoutLockfile_MCPToolsByMode(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + const listToolsRequest = `{"jsonrpc":"2.0", "method": "tools/list", "id": 1}` + "\n" + + defaultResults, err := cmdtest.NewScenarioTest("advanced", "mcp"). + Locally().WithStdin(strings.NewReader(listToolsRequest)).Run(t) + require.NoError(t, err) + require.Zero(t, defaultResults.ExitCode) + assert.Contains(t, defaultResults.Stdout, `"component-history"`) + + previewResults, err := cmdtest.NewScenarioTest("--without-lockfile", "advanced", "mcp"). + Locally().WithStdin(strings.NewReader(listToolsRequest)).Run(t) + require.NoError(t, err) + require.Zero(t, previewResults.ExitCode) + assert.NotContains(t, previewResults.Stdout, `"component-history"`) + assert.Contains(t, previewResults.Stdout, `"component-changed"`) +}