diff --git a/repo_cache/README.md b/repo_cache/README.md index 56044f5..2f30f28 100644 --- a/repo_cache/README.md +++ b/repo_cache/README.md @@ -35,7 +35,7 @@ uvx --from "git+https://github.com/eclipse-score/score_tools#subdirectory=repo_c ```bash score-repo-cache list --org eclipse-score -score-repo-cache sync --org eclipse-score --repo score --repo score_tools +score-repo-cache sync --org eclipse-score --repo score --repo 'score_*' ``` `sync` clones each selected repository's default branch into @@ -43,7 +43,10 @@ score-repo-cache sync --org eclipse-score --repo score --repo score_tools and resets an existing checkout back to a clean state if it was already cloned there. Repositories with no Git references are reported as empty and do not make the command fail; checkout, authentication, and other operational -errors remain failures. +errors remain failures. `--repo` values without `*`, `?`, or `[` are exact +names; values containing those characters use case-sensitive Python +`fnmatch` semantics. A pattern matching no repository eligible for +synchronization is reported separately from an exact name that is absent. ## Library diff --git a/repo_cache/src/cli.py b/repo_cache/src/cli.py index 7fadf60..b515b1e 100644 --- a/repo_cache/src/cli.py +++ b/repo_cache/src/cli.py @@ -59,7 +59,10 @@ def create_parser() -> argparse.ArgumentParser: action="append", default=None, metavar="NAME", - help="Exact repository name to include. Repeat to include more repositories.", + help=( + "Repository name or shell-style glob pattern to include. Repeat to " + "include more repositories." + ), ) sync_command.add_argument( "--cache-dir", diff --git a/repo_cache/src/sync.py b/repo_cache/src/sync.py index 1b317e5..3d4c70e 100644 --- a/repo_cache/src/sync.py +++ b/repo_cache/src/sync.py @@ -19,6 +19,7 @@ from collections.abc import Callable, Sequence from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass +from fnmatch import fnmatchcase from pathlib import Path from .checkout import sync_default_branch @@ -64,19 +65,31 @@ def sync_org( repos: Sequence[str] = (), include_archived: bool = False, workers: int = DEFAULT_SYNC_WORKERS, + max_selected_repositories: int | None = None, progress: Callable[[str], None] | None = None, ) -> SyncReport: """List an organization's repositories and sync each into `cache_dir/org/`. Raises RepoCacheError for an authentication failure or an unknown `repos` - name. Per-repository sync failures are captured in `SyncOutcome.error` - rather than raised, so one broken repository does not abort the rest. + name or pattern. Repository names without glob metacharacters are matched + exactly. Names containing ``*``, ``?``, or ``[`` use case-sensitive + :func:`fnmatch.fnmatchcase` matching against repositories eligible for + synchronization. Per-repository sync failures are captured in + `SyncOutcome.error` rather than raised, so one broken repository does not + abort the rest. Empty repositories are reported in `SyncReport.empty_repositories` instead of being treated as failures. + + ``max_selected_repositories`` limits the number of repositories selected + after exact and pattern matching. The limit is checked before any checkout + synchronization starts, which lets callers protect operations that are + only safe for a single repository. """ if workers < 1: raise RepoCacheError("sync worker count must be at least 1") + if max_selected_repositories is not None and max_selected_repositories < 1: + raise RepoCacheError("maximum selected repository count must be at least 1") report_progress = progress or (lambda _: None) @@ -84,28 +97,57 @@ def sync_org( ensure_authenticated() repositories = list_repositories(org=org) - active_repositories = tuple( + eligible_repositories = tuple( repository for repository in repositories if include_archived or not repository.archived ) - requested = set(repos) - available = {repository.name for repository in active_repositories} - missing = sorted(requested - available) - if missing: - raise RepoCacheError( - f"repository filter not found in organization: {', '.join(missing)}" + available = {repository.name for repository in eligible_repositories} + exact_names = {name for name in repos if not _has_glob_pattern(name)} + patterns = tuple(name for name in repos if _has_glob_pattern(name)) + missing = sorted(exact_names - available) + unmatched_patterns = tuple( + sorted( + { + pattern + for pattern in patterns + if not any(fnmatchcase(name, pattern) for name in available) + } ) - - report_progress(f"Found {len(active_repositories)} active repositories.") - report_progress(f"Using checkout cache at {cache_dir}.") + ) + if missing or unmatched_patterns: + diagnostics = [] + if missing: + diagnostics.append( + f"repository filter not found in organization: {', '.join(missing)}" + ) + if unmatched_patterns: + diagnostics.append( + "repository pattern matched no repositories eligible for synchronization: " + + ", ".join(unmatched_patterns) + ) + raise RepoCacheError("; ".join(diagnostics)) selected_repositories = tuple( repository - for repository in active_repositories - if not requested or repository.name in requested + for repository in eligible_repositories + if not repos + or repository.name in exact_names + or any(fnmatchcase(repository.name, pattern) for pattern in patterns) ) + if ( + max_selected_repositories is not None + and len(selected_repositories) > max_selected_repositories + ): + raise RepoCacheError( + f"repository selection matched {len(selected_repositories)} repositories; " + f"at most {max_selected_repositories} allowed" + ) + + report_progress(f"Found {len(eligible_repositories)} eligible repositories.") + report_progress(f"Using checkout cache at {cache_dir}.") + repositories_with_branches = tuple( repository for repository in selected_repositories @@ -162,3 +204,9 @@ def sync_org( outcomes[repository.name] for repository in selected_repositories ) return SyncReport(org=org, cache_dir=cache_dir, outcomes=ordered_outcomes) + + +def _has_glob_pattern(value: str) -> bool: + """Return whether ``value`` uses one of the supported fnmatch metacharacters.""" + + return any(character in value for character in "*?[") diff --git a/repo_cache/tests/test_sync.py b/repo_cache/tests/test_sync.py index ba3e882..5d996ed 100644 --- a/repo_cache/tests/test_sync.py +++ b/repo_cache/tests/test_sync.py @@ -146,6 +146,123 @@ def test_sync_org_rejects_an_unknown_repository_filter( sync_org(org="acme", cache_dir=tmp_path, repos=("missing",)) +def test_sync_org_expands_repository_patterns_before_checkout( + monkeypatch, tmp_path: Path +) -> None: + repositories = ( + Repository("score_tools", "main"), + Repository("score_checker", "main"), + Repository("other", "main"), + ) + _stub_listing(monkeypatch, repositories) + synced: list[str] = [] + monkeypatch.setattr( + sync_module, + "sync_default_branch", + lambda *, repository, branch, destination: synced.append(repository), + ) + + report = sync_org(org="acme", cache_dir=tmp_path, repos=("score_*",)) + + assert synced == ["acme/score_tools", "acme/score_checker"] + assert [outcome.repository.name for outcome in report.outcomes] == [ + "score_tools", + "score_checker", + ] + + +def test_sync_org_supports_fnmatch_character_classes_in_repository_patterns( + monkeypatch, tmp_path: Path +) -> None: + repositories = ( + Repository("vsps_a", "main"), + Repository("vsps_b", "main"), + Repository("vsps_c", "main"), + ) + _stub_listing(monkeypatch, repositories) + synced: list[str] = [] + monkeypatch.setattr( + sync_module, + "sync_default_branch", + lambda *, repository, branch, destination: synced.append(repository), + ) + + sync_org(org="acme", cache_dir=tmp_path, repos=("vsps_[ab]",)) + + assert synced == ["acme/vsps_a", "acme/vsps_b"] + + +def test_sync_org_distinguishes_an_unmatched_pattern_from_an_unknown_exact_name( + monkeypatch, tmp_path: Path +) -> None: + _stub_listing(monkeypatch, (Repository("score_tools", "main", archived=True),)) + + with pytest.raises(RepoCacheError) as error: + sync_org( + org="acme", + cache_dir=tmp_path, + repos=("missing", "unknown-*"), + include_archived=True, + ) + + assert str(error.value) == ( + "repository filter not found in organization: missing; " + "repository pattern matched no repositories eligible for synchronization: " + "unknown-*" + ) + + +def test_sync_org_checks_the_selection_limit_before_checkout( + monkeypatch, tmp_path: Path +) -> None: + repositories = ( + Repository("score_one", "main"), + Repository("score_two", "main"), + ) + _stub_listing(monkeypatch, repositories) + synced: list[str] = [] + monkeypatch.setattr( + sync_module, + "sync_default_branch", + lambda *, repository, branch, destination: synced.append(repository), + ) + + with pytest.raises( + RepoCacheError, + match="repository selection matched 2 repositories; at most 1 allowed", + ): + sync_org( + org="acme", + cache_dir=tmp_path, + repos=("score_*",), + max_selected_repositories=1, + ) + + assert synced == [] + + +def test_sync_org_counts_archived_repositories_in_selection_limits( + monkeypatch, tmp_path: Path +) -> None: + repositories = ( + Repository("score_one", "main"), + Repository("score_two", "main", archived=True), + ) + _stub_listing(monkeypatch, repositories) + + with pytest.raises( + RepoCacheError, + match="repository selection matched 2 repositories; at most 1 allowed", + ): + sync_org( + org="acme", + cache_dir=tmp_path, + repos=("score_*",), + include_archived=True, + max_selected_repositories=1, + ) + + def test_sync_org_rejects_fewer_than_one_worker(tmp_path: Path) -> None: with pytest.raises(RepoCacheError, match="at least 1"): sync_org(org="acme", cache_dir=tmp_path, workers=0) diff --git a/repo_policy_sync/README.md b/repo_policy_sync/README.md index 9d8a11a..6495950 100644 --- a/repo_policy_sync/README.md +++ b/repo_policy_sync/README.md @@ -75,6 +75,17 @@ uv run score-repo-policy-sync plan \ --repo reference_integration ``` +`--repo` and the TOML `repos` values accept exact names and case-sensitive +Python `fnmatch` patterns. Use `*`, `?`, or bracket expressions such as +`[ab]`; quote patterns in shell commands: + +```bash +uv run score-repo-policy-sync plan \ + --org eclipse-score \ + --repo 'score*' \ + --repo 'vsps_?' +``` + To exclude a policy for a repository or rollout: ```bash diff --git a/repo_policy_sync/docs/how-to/run-a-policy.md b/repo_policy_sync/docs/how-to/run-a-policy.md index 2c34085..92e2842 100644 --- a/repo_policy_sync/docs/how-to/run-a-policy.md +++ b/repo_policy_sync/docs/how-to/run-a-policy.md @@ -57,7 +57,8 @@ policy directories and command-line overrides. Limit a rollout to selected policy and repository names with repeatable `--policy` and `--repo` options. `--policy` selects an exact allowlist across -local and bundled policies; no other policy is run: +local and bundled policies; no other policy is run. Repository selections may +be exact names or case-sensitive Python `fnmatch` patterns: ```bash uv run score-repo-policy-sync plan \ @@ -67,6 +68,16 @@ uv run score-repo-policy-sync plan \ --policy minimum-bazel-version ``` +For example, quote a shell pattern to include all repositories with a common +prefix: + +```bash +uv run score-repo-policy-sync plan \ + --org eclipse-score \ + --repo 'score*' \ + --policy minimum-bazel-version +``` + When the plan is reviewed, apply the same selection to create or update the policy-owned pull requests: @@ -138,8 +149,9 @@ tool's ownership marker. Plan mode leaves the PR open. A changed or missing marker stops the run without closing the PR so it can be reviewed manually. Use `apply --recreate` only to rebuild one existing policy pull request from -the current default branch. It requires exactly one `--repo` and one -`--policy`; see the [CLI reference](../reference/cli.md) for all constraints. +the current default branch. It requires exactly one `--repo` selection, one +repository after any pattern is expanded, and one `--policy`; see the [CLI +reference](../reference/cli.md) for all constraints. ## Recovering from failures diff --git a/repo_policy_sync/docs/reference/cli.md b/repo_policy_sync/docs/reference/cli.md index 8f0f668..eb591f6 100644 --- a/repo_policy_sync/docs/reference/cli.md +++ b/repo_policy_sync/docs/reference/cli.md @@ -36,7 +36,7 @@ are CLI-only. | --- | --- | | `--org NAME` | GitHub organization to scan. May be set in TOML. | | `--policy NAME` | Select a policy by directory name from local or bundled policies. Repeat to select more than one; when present, only the selected policies run. Defaults to all local and bundled policies. | -| `--repo NAME` | Restrict the run to an exact repository name. Repeat to select more than one. | +| `--repo NAME` | Restrict the run to a repository name or shell-style glob pattern. Repeat to select more than one. | | *(stdout)* | Always prints the terminal policy-evaluation table; its status column shows policy pull-request state and number instead of the plain compliance status when that state is actionable. | ## Rare @@ -48,7 +48,7 @@ are CLI-only. | `--markdown-output PATH` | Also write the Markdown report to `PATH`. | | `--policy-dir PATH` | Local policy directory. Repeat to combine directories. Defaults to `./policies` in the current working directory when present. | | `--exclude-policy NAME` | Exclude one local or bundled policy. Applied after any explicit `--policy` selection; repeat to exclude more than one. | -| `--recreate` | On `apply`, rebuild one existing policy-owned pull request from its repository's current default branch. Requires exactly one `--repo` and exactly one `--policy`. | +| `--recreate` | On `apply`, rebuild one existing policy-owned pull request from its repository's current default branch. Requires exactly one `--repo` selection, exactly one selected repository after pattern expansion, and exactly one `--policy`. | | `--allow-dirty-pr`, `--no-allow-dirty-pr` | After the automatic formatting-fix retry, commit and push changes even if pre-commit still fails; create or keep the pull request as a draft and add a comment with the failure. | | `--quiet`, `--no-quiet` | Suppress progress messages on standard error. The report remains on standard output. | @@ -63,6 +63,17 @@ are CLI-only. Archived repositories are excluded from every run. Selecting an archived repository with `--repo` fails validation instead of silently ignoring it. +Repository selections use case-sensitive Python `fnmatch` semantics when the +value contains `*`, `?`, or `[`; otherwise the value is an exact repository +name. `*` matches any sequence of characters, `?` matches one character, and +bracket expressions such as `[ab]` or `[!ab]` match one character from or not +from the specified set. Quote patterns passed through a shell, for example +`--repo 'score*'`. A pattern that matches no repository eligible for +synchronization is reported as an unmatched pattern, separately from an exact +name that is absent. For +`--recreate`, the repository count is checked after pattern expansion and +before any checkout begins. + ## Exit status | Status | Meaning | diff --git a/repo_policy_sync/docs/reference/configuration.md b/repo_policy_sync/docs/reference/configuration.md index f095dc5..efdb4fa 100644 --- a/repo_policy_sync/docs/reference/configuration.md +++ b/repo_policy_sync/docs/reference/configuration.md @@ -64,3 +64,12 @@ directory exists. Setting it to `[]` disables local policy directories. `exclude_policies` accepts local or bundled policy directory names. When an option is present on the command line, its value replaces the corresponding TOML value, including list-valued options. Unknown policy names and unknown TOML fields are errors. + +Values in `repos` use the same repository selection rules as `--repo`: values +without `*`, `?`, or `[` are exact names, while values containing those +characters use case-sensitive Python `fnmatch` semantics. For example, +`repos = ["score*"]` selects every active repository whose name starts with +`score`. A pattern that matches no repository eligible for synchronization is +diagnosed separately from an exact repository name that is not present. When +`--recreate` is used, the expanded selection must contain exactly one +repository before checkout. diff --git a/repo_policy_sync/src/cli.py b/repo_policy_sync/src/cli.py index b489f7c..95006ba 100644 --- a/repo_policy_sync/src/cli.py +++ b/repo_policy_sync/src/cli.py @@ -93,7 +93,10 @@ def _add_common_arguments( "--repo", action="append", default=None, - help="Exact repository name to include. Repeat to include more repositories.", + help=( + "Repository name or shell-style glob pattern to include. Repeat to " + "include more repositories." + ), ) rare.add_argument( "--config", diff --git a/repo_policy_sync/src/runner.py b/repo_policy_sync/src/runner.py index 7362ae3..2a82a59 100644 --- a/repo_policy_sync/src/runner.py +++ b/repo_policy_sync/src/runner.py @@ -206,6 +206,7 @@ def run_policies( cache_dir=checkout_cache_directory, repos=repository_names, workers=sync_workers, + max_selected_repositories=1 if recreate else None, progress=report_progress, ) except RepoCacheError as exc: @@ -214,6 +215,11 @@ def run_policies( selected_repositories = tuple( outcome.repository for outcome in sync_report.outcomes ) + if recreate and len(selected_repositories) != 1: + raise RepoPolicySyncError( + "--recreate requires exactly one repository after repository " + "patterns are expanded" + ) skipped_repositories = { outcome.repository.name for outcome in sync_report.outcomes diff --git a/repo_policy_sync/tests/test_cli.py b/repo_policy_sync/tests/test_cli.py index f5af302..74cdaa9 100644 --- a/repo_policy_sync/tests/test_cli.py +++ b/repo_policy_sync/tests/test_cli.py @@ -526,6 +526,44 @@ def test_config_values_are_overridden_by_explicit_cli_values( assert any(path.parent.name == "minimum-bazel-version" for path in loaded_paths) +def test_repository_patterns_are_forwarded_from_cli_and_toml( + monkeypatch, tmp_path: Path +) -> None: + config_path = tmp_path / "config.toml" + config_path.write_text( + """[score-repo-policy-sync] +org = "eclipse-score" +repos = ["score*"] +""", + encoding="utf-8", + ) + observed = {} + monkeypatch.setattr(cli, "load_policies", lambda _: ()) + monkeypatch.setattr( + cli, + "run_policies", + lambda **kwargs: observed.update(kwargs) or _empty_report(), + ) + + assert cli.main(("plan", "--config", str(config_path), "--quiet")) == 0 + assert observed["repository_names"] == ("score*",) + + assert ( + cli.main( + ( + "plan", + "--config", + str(config_path), + "--repo", + "vsps_?", + "--quiet", + ) + ) + == 0 + ) + assert observed["repository_names"] == ("vsps_?",) + + def test_configurable_defaults_are_left_unset_for_config_merging() -> None: args = cli.create_parser().parse_args(("plan", "--org", "eclipse-score")) diff --git a/repo_policy_sync/tests/test_runner.py b/repo_policy_sync/tests/test_runner.py index 41a3697..f0153db 100644 --- a/repo_policy_sync/tests/test_runner.py +++ b/repo_policy_sync/tests/test_runner.py @@ -82,6 +82,7 @@ def fake_sync_org( repos=(), include_archived: bool = False, workers: int = 1, + max_selected_repositories: int | None = None, progress=None, ) -> SyncReport: report_progress = progress or (lambda _: None) @@ -102,6 +103,14 @@ def fake_sync_org( for repository in active if not requested or repository.name in requested ) + if ( + max_selected_repositories is not None + and len(selected) > max_selected_repositories + ): + raise RepoCacheError( + f"repository selection matched {len(selected)} repositories; " + f"at most {max_selected_repositories} allowed" + ) with_branches = tuple( repository for repository in selected @@ -632,6 +641,46 @@ def fake_sync_org(**_: object) -> SyncReport: ) +def test_runner_rejects_recreate_when_a_repository_pattern_selects_multiple( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "repository" + source.mkdir() + client = FakeRepositoryClient( + source, + (Repository("score-one", "main"), Repository("score-two", "main")), + ) + report = SyncReport( + org="eclipse-score", + cache_dir=tmp_path / "cache", + outcomes=( + SyncOutcome( + client.repositories[0], + tmp_path / "cache" / "eclipse-score" / "score-one", + ), + SyncOutcome( + client.repositories[1], + tmp_path / "cache" / "eclipse-score" / "score-two", + ), + ), + ) + monkeypatch.setattr(runner, "sync_org", lambda **_: report) + + with pytest.raises( + RepoPolicySyncError, + match="exactly one repository after repository patterns are expanded", + ): + run_policies( + client=client, + org="eclipse-score", + policies=(), + repository_names=("score-*",), + checkout_cache_directory=tmp_path / "cache", + apply=True, + recreate=True, + ) + + def test_runner_counts_a_sync_failure_once_per_repository( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: