diff --git a/README.md b/README.md index 83dd532..c56a354 100644 --- a/README.md +++ b/README.md @@ -98,3 +98,12 @@ To recover a partially completed delivery: The run summary records the source/tag/digest, output ref and main readback, and whether delivery advanced main or created a historical ref. A native installation proves client packaging only; authenticated read-only aggregate MCP access still requires the separate connection described above. Private docs and cloud deployment follow their own authorized workflows. MIT licensed; see [LICENSE](LICENSE). + + +## Public skill archives + +New canonical builds run the application's `cmd/skillpack` at the exact source commit. It calls the same `skills.PublicIndex` and `skills.PublicArchive` functions used by the runtime; this repository does not serialize a second ZIP format or author a second procedure. The native ZIP includes `public/index.json`, five deterministic ZIPs, and `public/source.json` with the same app and packager identities. Each per-skill ZIP contains exactly `SKILL.md` and `references/procedure.md`, unchanged, with raw-byte SHA-256 in the v0.2 index. Contributor skills and agent-specific native metadata do not enter those two-file archives. + +The existing signed application release candidate authenticates the enclosing native package digest. The receiver accepts either the complete legacy package or the complete new public-artifact inventory, verifies regular files, bounded sizes, exact member content and digests, and preserves immutable tags and prior releases. Downloaded packages remain data: the receiver never executes the producer or package content. Marketing mirrors those exact verified bytes at the versioned URLs in the index. An empty website index means no capable release has been synchronized yet. + +Activation requires publishing a reviewed packager revision and updating the application's immutable packager pin through the normal release workflow; these source changes do not publish or deploy that revision. Historical packages without the new public inventory stay valid. diff --git a/scripts/package.py b/scripts/package.py index 0107f46..a5cda32 100644 --- a/scripts/package.py +++ b/scripts/package.py @@ -3,6 +3,8 @@ import argparse import hashlib +import io +import tarfile import json from pathlib import Path import re @@ -26,6 +28,57 @@ "plugin.json", ".claude-plugin/plugin.json", "source.json", "README.md" }} | {"LICENSE", ".claude-plugin/marketplace.json", ".agents/plugins/marketplace.json"} +PUBLIC = {"public/index.json", "public/source.json"} | {f"public/{name}.zip" for name in SKILLS} + +# Go emits this fixed UTC modification timestamp; older artifacts omit it. +PUBLIC_MODIFIED_EXTRA = bytes.fromhex("555405000100a6ce12") + + +def generated(root): + return GENERATED | ({f"plugins/{NAME}/{path}" for path in PUBLIC} + if (Path(root) / "plugins" / NAME / "public").exists() else set()) + + +def produce_public(source_root, source_commit, output, version): + # Execute the canonical producer only during trusted build, never in the release receiver. + with tempfile.TemporaryDirectory(prefix="hitkeep-skillpack-source-") as directory: + source = Path(directory) + raw = git(source_root, "archive", source_commit, "--", "go.mod", "go.sum", "appurl", "skills", "cmd/skillpack") + with tarfile.open(fileobj=io.BytesIO(raw)) as archive: + archive.extractall(source, filter="data") + subprocess.run(["go", "run", "./cmd/skillpack", "--output", str(output), + "--base-url", f"https://hitkeep.com/skills/v{version}"], cwd=source, check=True) + + +def verify_public(plugin, version): + public = plugin / "public" + index = json.loads((public / "index.json").read_bytes()) + source = json.loads((plugin / "source.json").read_bytes()) + if (json.loads((public / "source.json").read_bytes()) != + {key: source[key] for key in ("repository", "commit", "packager_commit")}): + raise ValueError("public artifact source identity mismatch") + if (set(index) != {"$schema", "skills"} or + index["$schema"] != "https://schemas.agentskills.io/discovery/0.2.0/schema.json" or + len(index["skills"]) != len(SKILLS) or {entry["name"] for entry in index["skills"]} != SKILLS): + raise ValueError("public artifact index inventory mismatch") + for entry in index["skills"]: + name = entry["name"] + raw = (public / (name + ".zip")).read_bytes() + if (set(entry) != {"name", "type", "description", "url", "digest"} or entry["type"] != "archive" or + not isinstance(entry["description"], str) or not entry["description"] or + entry["url"] != f"https://hitkeep.com/skills/v{version}/{name}.zip" or + entry["digest"] != "sha256:" + hashlib.sha256(raw).hexdigest()): + raise ValueError("public artifact index entry mismatch") + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + if archive.namelist() != ["SKILL.md", "references/procedure.md"] or archive.comment: + raise ValueError("public artifact archive inventory mismatch") + for item in archive.infolist(): + if ((item.external_attr >> 16) != 0o100644 or item.file_size > 1024 * 1024 or + item.compress_type != zipfile.ZIP_STORED or item.flag_bits != 8 or item.extra not in (b"", PUBLIC_MODIFIED_EXTRA) or item.comment or + item.date_time != (1980, 1, 1, 0, 0, 0) or + archive.read(item) != (plugin / "skills" / name / item.filename).read_bytes()): + raise ValueError("public artifact differs from canonical source") + def git(root, *args): result = subprocess.run(["git", "-C", str(root), *args], capture_output=True) @@ -74,7 +127,7 @@ def sync(source_root: Path, source_commit: str, version: str, root: Path = ROOT) if root.is_symlink(): raise ValueError("symlink in output root") root = root.resolve() - for relative in GENERATED: + for relative in GENERATED | {f"plugins/{NAME}/{path}" for path in PUBLIC}: path = root / relative if any(parent.is_symlink() for parent in (path, *path.parents)): raise ValueError("symlink in output path") @@ -104,8 +157,14 @@ def sync(source_root: Path, source_commit: str, version: str, root: Path = ROOT) "packager_commit": packager_commit, "sha256": {path: hashlib.sha256(data).hexdigest() for path, data in files.items()}, }) + if (plugin / "public").exists(): + shutil.rmtree(plugin / "public") + produce_public(source_root, source_commit, plugin / "public", version) + write_json(plugin / "public/source.json", {key: json.loads((plugin / "source.json").read_text())[key] + for key in ("repository", "commit", "packager_commit")}) verify_source(source_root, source_commit, stage) - for relative in sorted(GENERATED): + for relative in sorted(generated(stage)): + (root / relative).parent.mkdir(parents=True, exist_ok=True) os.replace(stage / relative, root / relative) @@ -155,6 +214,8 @@ def verify(root=ROOT): if set(source["sha256"]) != expected: raise ValueError("canonical source allowlist changed") allowed = expected | {"plugin.json", ".claude-plugin/plugin.json", "source.json", "README.md"} + if (plugin / "public").exists(): + allowed |= PUBLIC allowed_dirs = {parent.as_posix() for relative in allowed for parent in Path(relative).parents if parent != Path(".")} actual = set() for path in plugin.rglob("*"): @@ -173,6 +234,8 @@ def verify(root=ROOT): for relative, digest in source["sha256"].items(): if hashlib.sha256((plugin / relative).read_bytes()).hexdigest() != digest: raise ValueError(f"canonical content changed: {relative}") + if (plugin / "public").exists(): + verify_public(plugin, version) return version, sorted(actual) diff --git a/scripts/release_delivery.py b/scripts/release_delivery.py index 0096654..06a363b 100644 --- a/scripts/release_delivery.py +++ b/scripts/release_delivery.py @@ -124,7 +124,9 @@ def unpack(raw, root): 'plugin.json', '.claude-plugin/plugin.json', 'source.json', 'README.md'}} with zipfile.ZipFile(io.BytesIO(raw)) as archive: entries = archive.infolist() - if len(entries) != len(expected) or {item.filename for item in entries} != expected: + names = {item.filename for item in entries} + public = {f'{package.NAME}/{path}' for path in package.PUBLIC} + if names not in (expected, expected | public) or len(entries) != len(names): raise ValueError('unsafe or incomplete package ZIP allowlist') if any((item.external_attr >> 16) != 0o100644 or item.file_size > 1024 * 1024 for item in entries): raise ValueError('non-regular, executable or oversized package ZIP content') @@ -203,7 +205,7 @@ def prepare(tag, sha, digest, root, temp): if package.verify(stage)[0] != tag[1:]: raise ValueError('package version differs from release') return dict(tag=tag, sha=sha, digest=digest, head=head, release=release, - candidate=mraw, tree={path: (stage / path).read_bytes() for path in package.GENERATED}) + candidate=mraw, tree={path: (stage / path).read_bytes() for path in package.generated(stage)}) def matching_tree(root, commit, files): @@ -213,10 +215,12 @@ def matching_tree(root, commit, files): if entry: metadata, path = entry.split(b'\t', 1) entries[path.decode()] = tuple(metadata.decode().split()[:2]) - if (set(files) != package.GENERATED or - entries != dict.fromkeys(package.GENERATED, ('100644', 'blob'))): + public = {f'plugins/{package.NAME}/{path}' for path in package.PUBLIC} + if (set(files) not in (package.GENERATED, package.GENERATED | public) or + set(entries) not in (package.GENERATED, package.GENERATED | public) or + any(mode != ('100644', 'blob') for mode in entries.values())): raise ValueError('generated tree membership or regular non-executable modes differ from allowlist') - actual = {path: package.git(root, 'show', commit + ':' + path) for path in package.GENERATED} + actual = {path: package.git(root, 'show', commit + ':' + path) for path in entries} with tempfile.TemporaryDirectory(prefix='hitkeep-assistant-tree-') as directory: stage = Path(directory) for relative, data in actual.items(): @@ -224,7 +228,7 @@ def matching_tree(root, commit, files): path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) package.verify(stage) - for path in package.GENERATED: + for path in set(actual) & set(files): if path.endswith('/marketplace.json'): current, expected = json.loads(actual[path]), json.loads(files[path]) if path == '.claude-plugin/marketplace.json': @@ -283,6 +287,8 @@ def write(state, root, temp): commit = head if not matches_main: command('git', '-C', str(gitroot), 'read-tree', head) + for path in sorted({f'plugins/{package.NAME}/{path}' for path in package.PUBLIC} - set(state['tree'])): + command('git', '-C', str(gitroot), 'update-index', '--force-remove', '--', path) for path, data in sorted(state['tree'].items()): blob = command('git', '-C', str(gitroot), 'hash-object', '-w', '--stdin', data=data).decode().strip() command('git', '-C', str(gitroot), 'update-index', '--add', '--cacheinfo', '100644,' + blob + ',' + path) @@ -293,7 +299,7 @@ def write(state, root, temp): commit = command('git', '-C', str(gitroot), 'commit-tree', tree, '-p', head, data=('chore: sync HitKeep ' + state['tag'] + '\n').encode(), env=env).decode().strip() paths = command('git', '-C', str(gitroot), 'diff-tree', '--no-commit-id', '--name-only', '-r', commit).decode().splitlines() - if not paths or not set(paths) <= package.GENERATED: + if not paths or not set(paths) <= set(state['tree']) | {f'plugins/{package.NAME}/{path}' for path in package.PUBLIC}: raise ValueError('generated diff changes non-allowlisted files') push = ('git', '-C', str(gitroot), '-c', 'credential.helper=', '-c', 'credential.helper=!gh auth git-credential', 'push', 'origin') if newest and not matches_main: diff --git a/tests/test_package.py b/tests/test_package.py index e52ef9f..6080b9d 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -6,6 +6,9 @@ import hashlib import tempfile import unittest +import io +import zipfile +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location("package", ROOT / "scripts/package.py") @@ -21,6 +24,10 @@ def setUp(self): for name in ("plugins", ".claude-plugin", ".agents"): shutil.copytree(ROOT / name, self.root / name) self.plugin = self.root / "plugins/hitkeep-analytics" + from test_release_delivery import fixture_public + mock = patch.object(package, "produce_public", side_effect=fixture_public) + mock.start() + self.addCleanup(mock.stop) def source(self): source = self.root / "upstream" @@ -36,6 +43,49 @@ def git(*args): git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "fixture") return source, git("rev-parse", "HEAD"), git + def test_public_archives_are_exact_deterministic_canonical_files(self): + source, commit, _ = self.source() + package.sync(source, commit, "2.16.0", self.root) + public = self.plugin / "public" + index = json.loads((public / "index.json").read_text()) + self.assertEqual(index["$schema"], "https://schemas.agentskills.io/discovery/0.2.0/schema.json") + self.assertEqual({entry["name"] for entry in index["skills"]}, package.SKILLS) + self.assertEqual({p.name for p in public.iterdir()}, {"index.json", "source.json"} | {name + ".zip" for name in package.SKILLS}) + self.assertEqual(json.loads((public / "source.json").read_text())["commit"], commit) + for entry in index["skills"]: + raw = (public / (entry["name"] + ".zip")).read_bytes() + self.assertEqual(entry["digest"], "sha256:" + hashlib.sha256(raw).hexdigest()) + self.assertEqual(entry["url"], "https://hitkeep.com/skills/v2.16.0/" + entry["name"] + ".zip") + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + self.assertEqual(archive.namelist(), ["SKILL.md", "references/procedure.md"]) + for item in archive.infolist(): + self.assertEqual(archive.read(item), (self.plugin / "skills" / entry["name"] / item.filename).read_bytes()) + self.assertEqual(item.compress_type, zipfile.ZIP_STORED) + self.assertEqual(item.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertEqual(item.external_attr >> 16, 0o100644) + self.assertEqual(item.flag_bits, 8) + self.assertEqual(item.extra, b"") + + (public / (index["skills"][0]["name"] + ".zip")).write_bytes(b"tampered") + with self.assertRaisesRegex(ValueError, "public artifact"): + package.verify(self.root) + + def test_public_timestamp_metadata_accepts_only_legacy_or_fixed_go_epoch(self): + from test_release_delivery import fixture_public + source, commit, _ = self.source() + fixed_epoch = bytes.fromhex("555405000100a6ce12") + for extra, accepted in [(b"", True), (fixed_epoch, True), (fixed_epoch[:-1] + b"\x13", False), (b"\x00\x00\x00\x00", False)]: + with self.subTest(extra=extra): + def produce(*args): + return fixture_public(*args, timestamp_extra=extra) + with patch.object(package, "produce_public", side_effect=produce): + if accepted: + package.sync(source, commit, "2.16.0", self.root) + package.verify(self.root) + else: + with self.assertRaisesRegex(ValueError, "public artifact"): + package.sync(source, commit, "2.16.0", self.root) + def test_rehashed_tamper_fails_independent_source_verification(self): source, commit, _ = self.source() package.sync(source, commit, "2.15.1", self.root) @@ -45,8 +95,8 @@ def test_rehashed_tamper_fails_independent_source_verification(self): data = json.loads(inventory.read_text()) data["sha256"]["skills/hitkeep-analytics/SKILL.md"] = hashlib.sha256(path.read_bytes()).hexdigest() inventory.write_text(json.dumps(data)) - package.verify(self.root) - with self.assertRaisesRegex(ValueError, "upstream content"): + # Rehashing the authored file cannot authorize a different archive or canonical source. + with self.assertRaisesRegex(ValueError, "public artifact|upstream content"): package.verify_source(source, commit, self.root) def test_sync_uses_immutable_source_and_updates_versions(self): @@ -113,7 +163,9 @@ def test_symlink_is_rejected(self): def test_mismatched_version_is_rejected(self): path = self.plugin / ".claude-plugin/plugin.json" - path.write_text(path.read_text().replace('"2.15.0"', '"0.0.0"')) + data = json.loads(path.read_text()) + data["version"] = "0.0.0" + path.write_text(json.dumps(data)) with self.assertRaisesRegex(ValueError, "manifest mismatch"): package.verify(self.root) diff --git a/tests/test_release_delivery.py b/tests/test_release_delivery.py index 063f243..8252e07 100644 --- a/tests/test_release_delivery.py +++ b/tests/test_release_delivery.py @@ -17,6 +17,34 @@ spec.loader.exec_module(delivery) + +def fixture_public(source_root, source_commit, output, version, timestamp_extra=b""): + output.mkdir() + entries = [] + class Stream(io.BytesIO): + def seek(self, *args): raise OSError("stream") + for name in sorted(delivery.package.SKILLS): + data = Stream() + with zipfile.ZipFile(data, "w") as archive: + for relative in ["SKILL.md", "references/procedure.md"]: + item = zipfile.ZipInfo(relative, (1980, 1, 1, 0, 0, 0)) + item.create_system = 3 + item.extra = timestamp_extra + item.external_attr = 0o100644 << 16 + archive.writestr(item, delivery.package.git(source_root, "show", f"{source_commit}:skills/{name}/{relative}")) + raw = data.getvalue() + (output / (name + ".zip")).write_bytes(raw) + entries.append(dict(name=name, type="archive", description="Fixture skill", url=f"https://hitkeep.com/skills/v{version}/{name}.zip", digest="sha256:" + hashlib.sha256(raw).hexdigest())) + (output / "index.json").write_text(json.dumps({"$schema":"https://schemas.agentskills.io/discovery/0.2.0/schema.json", "skills":entries})) + + + +_original_sync = delivery.package.sync +def _fixture_sync(*args, **kwargs): + with patch.object(delivery.package, "produce_public", side_effect=fixture_public): + return _original_sync(*args, **kwargs) + + class APITransportTests(unittest.TestCase): def test_repository_metadata_uses_canonical_api_root(self): with patch.object(delivery, 'command', return_value=b'{}') as command: @@ -163,6 +191,15 @@ def setUp(self): for name in ('plugins', '.claude-plugin', '.agents'): shutil.copytree(ROOT / name, self.root / name) shutil.copyfile(ROOT / 'LICENSE', self.root / 'LICENSE') + shutil.rmtree(self.root / 'plugins/hitkeep-analytics/public', ignore_errors=True) + # Keep the baseline historical even after the checked-in distribution advances. + for relative in ('plugins/hitkeep-analytics/plugin.json', 'plugins/hitkeep-analytics/.claude-plugin/plugin.json'): + path = self.root / relative + data = json.loads(path.read_text()); data['version'] = '2.15.0' + path.write_text(json.dumps(data)) + path = self.root / '.claude-plugin/marketplace.json' + data = json.loads(path.read_text()); data['plugins'][0]['version'] = '2.15.0' + path.write_text(json.dumps(data)) (self.root / 'reviewed-code').write_text('must not change') self.git(self.root, 'init', '-q', '-b', 'main') self.git(self.root, 'add', '-f', '.') @@ -356,6 +393,39 @@ def test_newest_fast_forwards_only_generated_files_and_retry_is_noop(self): self.assertEqual(previous, self.mutations) self.assertEqual(self.releases['v2.15.0']['assets'], [dict(name='preserved.zip')]) + def test_public_artifacts_publish_once_and_legacy_backport_keeps_its_shape(self): + state = self.new_state('v2.16.0') + tree = state['tree'] + prefix = 'plugins/' + delivery.package.NAME + '/' + entries = [] + class Stream(io.BytesIO): + def seek(self, *args): raise OSError('stream') + for name in sorted(delivery.package.SKILLS): + raw = Stream() + with zipfile.ZipFile(raw, 'w') as archive: + for relative in ['SKILL.md', 'references/procedure.md']: + entry = zipfile.ZipInfo(relative, (1980, 1, 1, 0, 0, 0)) + entry.create_system = 3 + entry.external_attr = 0o100644 << 16 + archive.writestr(entry, tree[prefix + f'skills/{name}/{relative}']) + data = raw.getvalue() + tree[prefix + 'public/' + name + '.zip'] = data + entries.append(dict(name=name, type='archive', description='Fixture', url=f'https://hitkeep.com/skills/v2.16.0/{name}.zip', digest='sha256:' + hashlib.sha256(data).hexdigest())) + source = json.loads(tree[prefix + 'source.json']) + tree[prefix + 'public/source.json'] = json.dumps({key: source[key] for key in ('repository', 'commit', 'packager_commit')}).encode() + tree[prefix + 'public/index.json'] = json.dumps({'$schema': 'https://schemas.agentskills.io/discovery/0.2.0/schema.json', 'skills': entries}).encode() + self.assertEqual(self.write(state), 'main-and-tag') + head = self.head() + state['head'] = head + self.assertEqual(self.write(state), 'main-and-tag') + self.assertEqual(self.head(), head) + legacy = self.new_state('v2.14.9') + self.assertEqual(self.write(legacy), 'historical-tag-only') + historical = self.tag(delivery.REPO, 'v2.14.9') + paths = self.git(self.remote, 'ls-tree', '-r', '--name-only', historical).splitlines() + self.assertFalse(any(path.startswith(prefix + 'public/') for path in paths)) + self.assertEqual(self.head(), head) + def test_backport_creates_only_historical_tag_and_link(self): self.write() head = self.head() @@ -429,7 +499,7 @@ def git(*args): generated.mkdir() for name in ('plugins', '.claude-plugin', '.agents'): shutil.copytree(ROOT / name, generated / name) - delivery.package.sync(self.source, self.sha, '2.15.1', generated) + _fixture_sync(self.source, self.sha, '2.15.1', generated) output = self.base / 'package.zip' delivery.package.package(output, generated) self.raw = output.read_bytes() @@ -499,7 +569,7 @@ def test_complete_verified_candidate_reaches_writer_once(self): report = delivery.deliver('v2.15.1', self.sha, self.digest, ROOT, writer=writer) writer.assert_called_once() self.assertEqual(report['disposition'], 'verified-fixture') - self.assertEqual(set(writer.call_args.args[0]['tree']), delivery.package.GENERATED) + self.assertEqual(set(writer.call_args.args[0]['tree']), delivery.package.GENERATED | {f'plugins/{delivery.package.NAME}/{path}' for path in delivery.package.PUBLIC}) def test_faults_never_reach_writer(self): scenarios = ('private', 'draft', 'prerelease', 'moved-tag', 'wrong-attempt', 'wrong-signer',