Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions repo_cache/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,18 @@ uvx --from "git+https://github.com/eclipse-score/score_tools#subdirectory=repo_c

```bash
score-repo-cache list --org eclipse-score
score-repo-cache sync --org eclipse-score --repo score --repo score_tools
score-repo-cache sync --org eclipse-score --repo score --repo 'score_*'
```

`sync` clones each selected repository's default branch into
`~/.cache/repo-cache/<org>/<name>` (override with `--cache-dir`), or fetches
and resets an existing checkout back to a clean state if it was already
cloned there. Repositories with no Git references are reported as empty and
do not make the command fail; checkout, authentication, and other operational
errors remain failures.
errors remain failures. `--repo` values without `*`, `?`, or `[` are exact
names; values containing those characters use case-sensitive Python
`fnmatch` semantics. A pattern matching no repository eligible for
synchronization is reported separately from an exact name that is absent.

## Library

Expand Down
5 changes: 4 additions & 1 deletion repo_cache/src/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,10 @@ def create_parser() -> argparse.ArgumentParser:
action="append",
default=None,
metavar="NAME",
help="Exact repository name to include. Repeat to include more repositories.",
help=(
"Repository name or shell-style glob pattern to include. Repeat to "
"include more repositories."
),
)
sync_command.add_argument(
"--cache-dir",
Expand Down
76 changes: 62 additions & 14 deletions repo_cache/src/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from collections.abc import Callable, Sequence
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from fnmatch import fnmatchcase
from pathlib import Path

from .checkout import sync_default_branch
Expand Down Expand Up @@ -64,48 +65,89 @@ def sync_org(
repos: Sequence[str] = (),
include_archived: bool = False,
workers: int = DEFAULT_SYNC_WORKERS,
max_selected_repositories: int | None = None,
progress: Callable[[str], None] | None = None,
) -> SyncReport:
"""List an organization's repositories and sync each into `cache_dir/org/<name>`.

Raises RepoCacheError for an authentication failure or an unknown `repos`
name. Per-repository sync failures are captured in `SyncOutcome.error`
rather than raised, so one broken repository does not abort the rest.
name or pattern. Repository names without glob metacharacters are matched
exactly. Names containing ``*``, ``?``, or ``[`` use case-sensitive
:func:`fnmatch.fnmatchcase` matching against repositories eligible for
synchronization. Per-repository sync failures are captured in
`SyncOutcome.error` rather than raised, so one broken repository does not
abort the rest.
Empty repositories are reported in `SyncReport.empty_repositories` instead
of being treated as failures.

``max_selected_repositories`` limits the number of repositories selected
after exact and pattern matching. The limit is checked before any checkout
synchronization starts, which lets callers protect operations that are
only safe for a single repository.
"""

if workers < 1:
raise RepoCacheError("sync worker count must be at least 1")
if max_selected_repositories is not None and max_selected_repositories < 1:
raise RepoCacheError("maximum selected repository count must be at least 1")

report_progress = progress or (lambda _: None)

report_progress("Checking gh authentication...")
ensure_authenticated()

repositories = list_repositories(org=org)
active_repositories = tuple(
eligible_repositories = tuple(
repository
for repository in repositories
if include_archived or not repository.archived
)

requested = set(repos)
available = {repository.name for repository in active_repositories}
missing = sorted(requested - available)
if missing:
raise RepoCacheError(
f"repository filter not found in organization: {', '.join(missing)}"
available = {repository.name for repository in eligible_repositories}
exact_names = {name for name in repos if not _has_glob_pattern(name)}
patterns = tuple(name for name in repos if _has_glob_pattern(name))
missing = sorted(exact_names - available)
unmatched_patterns = tuple(
sorted(
{
pattern
for pattern in patterns
if not any(fnmatchcase(name, pattern) for name in available)
}
)

report_progress(f"Found {len(active_repositories)} active repositories.")
report_progress(f"Using checkout cache at {cache_dir}.")
)
if missing or unmatched_patterns:
diagnostics = []
if missing:
diagnostics.append(
f"repository filter not found in organization: {', '.join(missing)}"
)
if unmatched_patterns:
diagnostics.append(
"repository pattern matched no repositories eligible for synchronization: "
+ ", ".join(unmatched_patterns)
)
raise RepoCacheError("; ".join(diagnostics))

selected_repositories = tuple(
repository
for repository in active_repositories
if not requested or repository.name in requested
for repository in eligible_repositories
if not repos
or repository.name in exact_names
or any(fnmatchcase(repository.name, pattern) for pattern in patterns)
)
if (
max_selected_repositories is not None
and len(selected_repositories) > max_selected_repositories
):
raise RepoCacheError(
f"repository selection matched {len(selected_repositories)} repositories; "
f"at most {max_selected_repositories} allowed"
)

report_progress(f"Found {len(eligible_repositories)} eligible repositories.")
report_progress(f"Using checkout cache at {cache_dir}.")

repositories_with_branches = tuple(
repository
for repository in selected_repositories
Expand Down Expand Up @@ -162,3 +204,9 @@ def sync_org(
outcomes[repository.name] for repository in selected_repositories
)
return SyncReport(org=org, cache_dir=cache_dir, outcomes=ordered_outcomes)


def _has_glob_pattern(value: str) -> bool:
"""Return whether ``value`` uses one of the supported fnmatch metacharacters."""

return any(character in value for character in "*?[")
117 changes: 117 additions & 0 deletions repo_cache/tests/test_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,123 @@ def test_sync_org_rejects_an_unknown_repository_filter(
sync_org(org="acme", cache_dir=tmp_path, repos=("missing",))


def test_sync_org_expands_repository_patterns_before_checkout(
monkeypatch, tmp_path: Path
) -> None:
repositories = (
Repository("score_tools", "main"),
Repository("score_checker", "main"),
Repository("other", "main"),
)
_stub_listing(monkeypatch, repositories)
synced: list[str] = []
monkeypatch.setattr(
sync_module,
"sync_default_branch",
lambda *, repository, branch, destination: synced.append(repository),
)

report = sync_org(org="acme", cache_dir=tmp_path, repos=("score_*",))

assert synced == ["acme/score_tools", "acme/score_checker"]
assert [outcome.repository.name for outcome in report.outcomes] == [
"score_tools",
"score_checker",
]


def test_sync_org_supports_fnmatch_character_classes_in_repository_patterns(
monkeypatch, tmp_path: Path
) -> None:
repositories = (
Repository("vsps_a", "main"),
Repository("vsps_b", "main"),
Repository("vsps_c", "main"),
)
_stub_listing(monkeypatch, repositories)
synced: list[str] = []
monkeypatch.setattr(
sync_module,
"sync_default_branch",
lambda *, repository, branch, destination: synced.append(repository),
)

sync_org(org="acme", cache_dir=tmp_path, repos=("vsps_[ab]",))

assert synced == ["acme/vsps_a", "acme/vsps_b"]


def test_sync_org_distinguishes_an_unmatched_pattern_from_an_unknown_exact_name(
monkeypatch, tmp_path: Path
) -> None:
_stub_listing(monkeypatch, (Repository("score_tools", "main", archived=True),))

with pytest.raises(RepoCacheError) as error:
sync_org(
org="acme",
cache_dir=tmp_path,
repos=("missing", "unknown-*"),
include_archived=True,
)

assert str(error.value) == (
"repository filter not found in organization: missing; "
"repository pattern matched no repositories eligible for synchronization: "
"unknown-*"
)


def test_sync_org_checks_the_selection_limit_before_checkout(
monkeypatch, tmp_path: Path
) -> None:
repositories = (
Repository("score_one", "main"),
Repository("score_two", "main"),
)
_stub_listing(monkeypatch, repositories)
synced: list[str] = []
monkeypatch.setattr(
sync_module,
"sync_default_branch",
lambda *, repository, branch, destination: synced.append(repository),
)

with pytest.raises(
RepoCacheError,
match="repository selection matched 2 repositories; at most 1 allowed",
):
sync_org(
org="acme",
cache_dir=tmp_path,
repos=("score_*",),
max_selected_repositories=1,
)

assert synced == []


def test_sync_org_counts_archived_repositories_in_selection_limits(
monkeypatch, tmp_path: Path
) -> None:
repositories = (
Repository("score_one", "main"),
Repository("score_two", "main", archived=True),
)
_stub_listing(monkeypatch, repositories)

with pytest.raises(
RepoCacheError,
match="repository selection matched 2 repositories; at most 1 allowed",
):
sync_org(
org="acme",
cache_dir=tmp_path,
repos=("score_*",),
include_archived=True,
max_selected_repositories=1,
)


def test_sync_org_rejects_fewer_than_one_worker(tmp_path: Path) -> None:
with pytest.raises(RepoCacheError, match="at least 1"):
sync_org(org="acme", cache_dir=tmp_path, workers=0)
Expand Down
11 changes: 11 additions & 0 deletions repo_policy_sync/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,17 @@ uv run score-repo-policy-sync plan \
--repo reference_integration
```

`--repo` and the TOML `repos` values accept exact names and case-sensitive
Python `fnmatch` patterns. Use `*`, `?`, or bracket expressions such as
`[ab]`; quote patterns in shell commands:

```bash
uv run score-repo-policy-sync plan \
--org eclipse-score \
--repo 'score*' \
--repo 'vsps_?'
```

To exclude a policy for a repository or rollout:

```bash
Expand Down
18 changes: 15 additions & 3 deletions repo_policy_sync/docs/how-to/run-a-policy.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ policy directories and command-line overrides.

Limit a rollout to selected policy and repository names with repeatable
`--policy` and `--repo` options. `--policy` selects an exact allowlist across
local and bundled policies; no other policy is run:
local and bundled policies; no other policy is run. Repository selections may
be exact names or case-sensitive Python `fnmatch` patterns:

```bash
uv run score-repo-policy-sync plan \
Expand All @@ -67,6 +68,16 @@ uv run score-repo-policy-sync plan \
--policy minimum-bazel-version
```

For example, quote a shell pattern to include all repositories with a common
prefix:

```bash
uv run score-repo-policy-sync plan \
--org eclipse-score \
--repo 'score*' \
--policy minimum-bazel-version
```

When the plan is reviewed, apply the same selection to create or update the
policy-owned pull requests:

Expand Down Expand Up @@ -138,8 +149,9 @@ tool's ownership marker. Plan mode leaves the PR open. A changed or missing
marker stops the run without closing the PR so it can be reviewed manually.

Use `apply --recreate` only to rebuild one existing policy pull request from
the current default branch. It requires exactly one `--repo` and one
`--policy`; see the [CLI reference](../reference/cli.md) for all constraints.
the current default branch. It requires exactly one `--repo` selection, one
repository after any pattern is expanded, and one `--policy`; see the [CLI
reference](../reference/cli.md) for all constraints.

## Recovering from failures

Expand Down
15 changes: 13 additions & 2 deletions repo_policy_sync/docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ are CLI-only.
| --- | --- |
| `--org NAME` | GitHub organization to scan. May be set in TOML. |
| `--policy NAME` | Select a policy by directory name from local or bundled policies. Repeat to select more than one; when present, only the selected policies run. Defaults to all local and bundled policies. |
| `--repo NAME` | Restrict the run to an exact repository name. Repeat to select more than one. |
| `--repo NAME` | Restrict the run to a repository name or shell-style glob pattern. Repeat to select more than one. |
| *(stdout)* | Always prints the terminal policy-evaluation table; its status column shows policy pull-request state and number instead of the plain compliance status when that state is actionable. |

## Rare
Expand All @@ -48,7 +48,7 @@ are CLI-only.
| `--markdown-output PATH` | Also write the Markdown report to `PATH`. |
| `--policy-dir PATH` | Local policy directory. Repeat to combine directories. Defaults to `./policies` in the current working directory when present. |
| `--exclude-policy NAME` | Exclude one local or bundled policy. Applied after any explicit `--policy` selection; repeat to exclude more than one. |
| `--recreate` | On `apply`, rebuild one existing policy-owned pull request from its repository's current default branch. Requires exactly one `--repo` and exactly one `--policy`. |
| `--recreate` | On `apply`, rebuild one existing policy-owned pull request from its repository's current default branch. Requires exactly one `--repo` selection, exactly one selected repository after pattern expansion, and exactly one `--policy`. |
| `--allow-dirty-pr`, `--no-allow-dirty-pr` | After the automatic formatting-fix retry, commit and push changes even if pre-commit still fails; create or keep the pull request as a draft and add a comment with the failure. |
| `--quiet`, `--no-quiet` | Suppress progress messages on standard error. The report remains on standard output. |

Expand All @@ -63,6 +63,17 @@ are CLI-only.
Archived repositories are excluded from every run. Selecting an archived
repository with `--repo` fails validation instead of silently ignoring it.

Repository selections use case-sensitive Python `fnmatch` semantics when the
value contains `*`, `?`, or `[`; otherwise the value is an exact repository
name. `*` matches any sequence of characters, `?` matches one character, and
bracket expressions such as `[ab]` or `[!ab]` match one character from or not
from the specified set. Quote patterns passed through a shell, for example
`--repo 'score*'`. A pattern that matches no repository eligible for
synchronization is reported as an unmatched pattern, separately from an exact
name that is absent. For
`--recreate`, the repository count is checked after pattern expansion and
before any checkout begins.

## Exit status

| Status | Meaning |
Expand Down
9 changes: 9 additions & 0 deletions repo_policy_sync/docs/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,12 @@ directory exists. Setting it to `[]` disables local policy directories.
`exclude_policies` accepts local or bundled policy directory names. When an option is present on the command line, its value replaces the
corresponding TOML value, including list-valued options. Unknown policy names
and unknown TOML fields are errors.

Values in `repos` use the same repository selection rules as `--repo`: values
without `*`, `?`, or `[` are exact names, while values containing those
characters use case-sensitive Python `fnmatch` semantics. For example,
`repos = ["score*"]` selects every active repository whose name starts with
`score`. A pattern that matches no repository eligible for synchronization is
diagnosed separately from an exact repository name that is not present. When
`--recreate` is used, the expanded selection must contain exactly one
repository before checkout.
5 changes: 4 additions & 1 deletion repo_policy_sync/src/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,10 @@ def _add_common_arguments(
"--repo",
action="append",
default=None,
help="Exact repository name to include. Repeat to include more repositories.",
help=(
"Repository name or shell-style glob pattern to include. Repeat to "
"include more repositories."
),
)
rare.add_argument(
"--config",
Expand Down
Loading