From b39d68e15d06fb7860e88e16fbd13734bf7d2e70 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 10:57:43 +0000 Subject: [PATCH 1/8] Record frozen editions as append-only files under vortex/editions A frozen edition carries a read-forever guarantee, so its encoding set must never change again. Enforcement so far was a single inline assertion pinning core2026.07.0 -- not the edition the default writer targets -- leaving three of five frozen core editions unpinned, and living in the same mutable tree as the declarations it guarded. Generate one TOML record per frozen edition, holding the identifier, the recorded min_vortex_version, and the full computed encoding set. A test keeps the records in step with EDITION_DECLARATIONS and rejects a record with no frozen edition behind it, so deleting or unfreezing a declaration fails too. Update mode never removes a file, so unfreezing cannot be laundered through the generator. Two CI checks close the loop. The generated-files job regenerates the records and fails if git is dirty, alongside the existing flatbuffers and proto generation. A new job rejects any diff that modifies, deletes, or renames a record, and any newly added edition that is not newer than its family's newest recorded edition. required_vortex_release is deliberately left out of the records: it is backfilled from compat-fixture evidence after an edition freezes, so pinning it would put that backfill in conflict with the append-only rule. Signed-off-by: "Joe Isaacs" --- .github/scripts/check_frozen_editions.py | 154 +++++++++++++++++ .github/workflows/ci.yml | 19 +++ vortex/editions/core2025.05.0.toml | 64 +++++++ vortex/editions/core2025.06.0.toml | 47 ++++++ vortex/editions/core2025.10.0.toml | 52 ++++++ vortex/editions/core2026.07.0.toml | 50 ++++++ vortex/editions/core2026.08.0.toml | 51 ++++++ vortex/src/editions/frozen.rs | 202 +++++++++++++++++++++++ vortex/src/editions/mod.rs | 2 + vortex/src/editions/tests.rs | 49 ------ 10 files changed, 641 insertions(+), 49 deletions(-) create mode 100644 .github/scripts/check_frozen_editions.py create mode 100644 vortex/editions/core2025.05.0.toml create mode 100644 vortex/editions/core2025.06.0.toml create mode 100644 vortex/editions/core2025.10.0.toml create mode 100644 vortex/editions/core2026.07.0.toml create mode 100644 vortex/editions/core2026.08.0.toml create mode 100644 vortex/src/editions/frozen.rs diff --git a/.github/scripts/check_frozen_editions.py b/.github/scripts/check_frozen_editions.py new file mode 100644 index 00000000000..57f4b4944a0 --- /dev/null +++ b/.github/scripts/check_frozen_editions.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Check that the frozen edition records under `vortex/editions` are append-only. + +A frozen edition carries a read-forever guarantee, so its record may never change: the only +legal edit to the directory is adding a file for a newly frozen edition, and that edition must +be newer than every edition already recorded for its family. + +Usage: + python3 check_frozen_editions.py --base origin/develop +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +RECORD_DIR = "vortex/editions" + +# `core2026.08.0.toml`: the file name is the edition id, so the record's identity is visible +# in the diff without reading the file. +RECORD_NAME = re.compile( + r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" +) + +EDITION_FIELD = re.compile(r'^edition = "(?P[^"]+)"$', re.MULTILINE) + +REMEDY = ( + "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" + " vortex/src/editions// and regenerate the records with\n" + " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`." +) + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], capture_output=True, text=True, check=False) + if result.returncode != 0: + sys.exit(f"git {' '.join(args)} failed:\n{result.stderr.strip()}") + return result.stdout + + +def merge_base(base: str) -> str: + result = subprocess.run( + ["git", "merge-base", base, "HEAD"], capture_output=True, text=True, check=False + ) + if result.returncode != 0: + sys.exit( + f"cannot find a merge base between {base} and HEAD:\n" + f"{result.stderr.strip()}\n" + "The checkout is probably too shallow; this check needs `fetch-depth: 0`." + ) + return result.stdout.strip() + + +def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: + """Split a record file name into its family and its chronological sort key.""" + match = RECORD_NAME.match(name) + if match is None: + sys.exit( + f"{RECORD_DIR}/{name} is not a valid record name.\n" + "Records are named after the edition they record, e.g. `core2026.08.0.toml`." + ) + return match["family"], (int(match["year"]), int(match["month"]), int(match["version"])) + + +def changed_records(base: str) -> list[tuple[str, list[str]]]: + """The status and paths of every change to the record directory since `base`.""" + raw = git("diff", "--name-status", "-z", base, "HEAD", "--", RECORD_DIR) + fields = [field for field in raw.split("\0") if field] + changes: list[tuple[str, list[str]]] = [] + index = 0 + while index < len(fields): + status = fields[index] + # Renames and copies carry both the old and the new path. + count = 2 if status[0] in ("R", "C") else 1 + changes.append((status, fields[index + 1 : index + 1 + count])) + index += 1 + count + return changes + + +def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: + """The newest edition already recorded for each family at `base`.""" + newest: dict[str, tuple[int, int, int]] = {} + listing = git("ls-tree", "-r", "--name-only", base, "--", RECORD_DIR) + for path in listing.splitlines(): + family, key = parse_name(Path(path).name) + newest[family] = max(key, newest.get(family, (0, 0, 0))) + return newest + + +def check(base: str) -> list[str]: + errors: list[str] = [] + added: list[str] = [] + + for status, paths in changed_records(base): + if status == "A": + added.extend(paths) + continue + verb = {"M": "modifies", "D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} + errors.append(f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}") + + newest = recorded_at(base) + for path in sorted(added): + name = Path(path).name + family, key = parse_name(name) + + previous = newest.get(family) + if previous is not None and key <= previous: + recorded = f"{previous[0]}.{previous[1]:02}.{previous[2]}" + errors.append( + f"adds {name}, which is not newer than the {family} edition already " + f"recorded ({family}{recorded}). Editions may only be added going forward." + ) + + # The file name is the edition's identity, so it has to agree with the content. + text = Path(path).read_text() + match = EDITION_FIELD.search(text) + if match is None: + errors.append(f"adds {name}, which has no `edition` field") + elif match["edition"] != name.removesuffix(".toml"): + errors.append( + f"adds {name}, which records edition {match['edition']!r}; the file name " + "must be the edition id" + ) + + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--base", + default="origin/develop", + help="the revision to compare against (default: origin/develop)", + ) + args = parser.parse_args() + + base = merge_base(args.base) + errors = check(base) + if not errors: + print(f"{RECORD_DIR} is append-only against {args.base} ({base[:12]}).") + return 0 + + print(f"This change breaks the frozen edition records in {RECORD_DIR}:\n", file=sys.stderr) + for error in errors: + print(f" - it {error}", file=sys.stderr) + print(f"\n{REMEDY}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67641d15e54..cee16d5a4d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,20 @@ jobs: -c .yamllint.yaml \ .github/ + frozen-editions: + name: "Frozen editions are append-only" + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # The check compares against the merge base, so it needs real history. + fetch-depth: 0 + - name: Check frozen edition records + run: | + BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" + python3 .github/scripts/check_frozen_editions.py --base "$BASE" + python-lint: name: "Python (lint)" runs-on: >- @@ -663,6 +677,11 @@ jobs: - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi + - name: "regenerate the frozen edition records" + env: + UPDATE_FROZEN_EDITIONS: "1" + run: | + cargo test --profile ci -p vortex --lib editions::frozen - name: "Make sure no files changed after regenerating" run: | git status --porcelain diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml new file mode 100644 index 00000000000..0faeeb2333a --- /dev/null +++ b/vortex/editions/core2025.05.0.toml @@ -0,0 +1,64 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2025.05.0" +family = "core" +min_vortex_version = "0.36.0" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml new file mode 100644 index 00000000000..2a584ba87e6 --- /dev/null +++ b/vortex/editions/core2025.06.0.toml @@ -0,0 +1,47 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2025.06.0" +family = "core" +min_vortex_version = "0.40.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.pco", + "vortex.sequence", + "vortex.zstd", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml new file mode 100644 index 00000000000..08c0688624c --- /dev/null +++ b/vortex/editions/core2025.10.0.toml @@ -0,0 +1,52 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2025.10.0" +family = "core" +min_vortex_version = "0.54.0" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.rle", + "vortex.fixed_size_list", + "vortex.listview", + "vortex.masked", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml new file mode 100644 index 00000000000..53e3f4c09ae --- /dev/null +++ b/vortex/editions/core2026.07.0.toml @@ -0,0 +1,50 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2026.07.0" +family = "core" +min_vortex_version = "0.65.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.variant", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml new file mode 100644 index 00000000000..c1920b095cf --- /dev/null +++ b/vortex/editions/core2026.08.0.toml @@ -0,0 +1,51 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2026.08.0" +family = "core" +min_vortex_version = "0.84.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.map", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/src/editions/frozen.rs b/vortex/src/editions/frozen.rs new file mode 100644 index 00000000000..f538e777d6c --- /dev/null +++ b/vortex/src/editions/frozen.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The frozen-edition record under `vortex/editions`. +//! +//! A frozen edition carries a read-forever guarantee, so its encoding set must never change +//! again. Every frozen edition has one generated TOML file recording that contract: the +//! identifier, the minimum Vortex version whose reader supports it, and the full encoding +//! set. Freezing a new edition adds a file; nothing else may touch the directory, which CI +//! enforces by rejecting any diff that modifies or deletes an existing record +//! (`.github/scripts/check_frozen_editions.py`). +//! +//! The test here keeps the record honest in the other direction: the files must match what +//! [`super::EDITION_DECLARATIONS`] actually computes, so a frozen edition cannot drift +//! without the record drifting with it. Regenerate after freezing a new edition with: +//! +//! ```bash +//! UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen +//! ``` +//! +//! Only facts that are frozen by freezing are recorded. In particular +//! [`vortex_edition::EditionInclusion::required_vortex_release`] is not: it is backfilled +//! from compat-fixture evidence long after an edition freezes, and pinning it here would put +//! that legitimate backfill in conflict with the append-only rule. + +use std::collections::BTreeSet; +use std::env; +use std::fs; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::anyhow; +use vortex_edition::Edition; +use vortex_edition::EditionError; +use vortex_edition::EditionSession; + +use super::EDITION_DECLARATIONS; + +/// Set to any value to rewrite the record instead of verifying it. +const UPDATE_VAR: &str = "UPDATE_FROZEN_EDITIONS"; + +const REGENERATE: &str = "UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen"; + +const HEADER: &str = "\ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI."; + +fn record_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("editions") +} + +fn session() -> Result { + let session = EditionSession::empty(); + for declaration in EDITION_DECLARATIONS { + session.declare(declaration)?; + } + Ok(session) +} + +/// The frozen editions, paired with the version that freezing recorded. Drafts have no +/// record: a file appears in the directory at the moment an edition freezes. +fn frozen(session: &EditionSession) -> Vec<(Edition, &'static str)> { + session + .editions() + .into_iter() + .filter_map(|edition| edition.min_vortex_version.map(|version| (edition, version))) + .collect() +} + +/// Render one edition's record. Deterministic: both encoding lists are sorted by id, so the +/// generated bytes depend only on the declarations. +fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) -> String { + let inclusions = session.encodings_in(&edition.id); + let members: BTreeSet<&str> = inclusions + .iter() + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + let added: BTreeSet<&str> = inclusions + .iter() + .filter(|inclusion| inclusion.since == edition.id) + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + + let list = |ids: &BTreeSet<&str>| -> Vec { + ids.iter().map(|id| format!(" \"{id}\",")).collect() + }; + + let mut lines = vec![ + HEADER.to_string(), + String::new(), + format!("edition = \"{}\"", edition.id), + format!("family = \"{}\"", edition.id.family), + format!("min_vortex_version = \"{min_vortex_version}\""), + String::new(), + "# The encodings that join the family at this edition.".to_string(), + "added = [".to_string(), + ]; + lines.extend(list(&added)); + lines.extend([ + "]".to_string(), + String::new(), + "# The edition's full membership: the encodings above, plus every member of earlier" + .to_string(), + "# editions of the family.".to_string(), + "encodings = [".to_string(), + ]); + lines.extend(list(&members)); + lines.extend(["]".to_string(), String::new()]); + lines.join("\n") +} + +fn record_path(dir: &std::path::Path, edition: &Edition) -> PathBuf { + dir.join(format!("{}.toml", edition.id)) +} + +/// The `*.toml` file names present in the record directory. +fn existing_records(dir: &std::path::Path) -> anyhow::Result> { + let mut names = BTreeSet::new(); + for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + names.insert(name.to_string()); + } + } + Ok(names) +} + +/// Every frozen edition has a record, every record matches the declarations exactly, and no +/// record exists without a frozen edition behind it. +/// +/// The third check is what catches a frozen edition being deleted or unfrozen, so the update +/// mode deliberately never removes a file: unfreezing cannot be laundered through the +/// generator. +#[test] +fn records_match_the_frozen_editions() -> anyhow::Result<()> { + let session = session()?; + let dir = record_dir(); + let update = env::var_os(UPDATE_VAR).is_some(); + + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + + let mut expected_names = BTreeSet::new(); + for (edition, min_vortex_version) in frozen(&session) { + let path = record_path(&dir, &edition); + let expected = record(&session, &edition, min_vortex_version); + expected_names.insert(format!("{}.toml", edition.id)); + + let actual = match fs::read_to_string(&path) { + Ok(actual) => Some(actual), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), + }; + + if actual.as_deref() == Some(expected.as_str()) { + continue; + } + if update { + fs::write(&path, &expected).with_context(|| format!("writing {}", path.display()))?; + continue; + } + + return Err(match actual { + Some(_) => anyhow!( + "the record of frozen edition {} no longer matches its declaration.\n\ + A frozen edition's encodings are fixed forever: declare a new edition \ + instead of changing this one.\n\ + If the edition is genuinely new, regenerate with `{REGENERATE}`.\n\ + Record: {}", + edition.id, + path.display(), + ), + None => anyhow!( + "frozen edition {} has no record. Regenerate with `{REGENERATE}`.\n\ + Expected: {}", + edition.id, + path.display(), + ), + }); + } + + let strays: Vec = existing_records(&dir)? + .difference(&expected_names) + .cloned() + .collect(); + if !strays.is_empty() { + return Err(anyhow!( + "{} has records with no frozen edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted or returned to draft; its declaration must \ + stay in `EDITION_DECLARATIONS` with its `min_vortex_version` recorded.", + dir.display(), + )); + } + + Ok(()) +} diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 88e4f351d0d..ab20470f2f7 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -15,6 +15,8 @@ pub mod core; #[cfg(test)] +mod frozen; +#[cfg(test)] mod tests; pub mod unstable; diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index b8608f4427a..46dcb73d53e 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -62,55 +62,6 @@ fn every_declared_edition_validates() -> Result<(), EditionError> { Ok(()) } -/// The full encoding set of the newest frozen `core` edition. This set is frozen: the only -/// way it may change is by declaring a *new* edition, so a failure here means a frozen -/// declaration was edited. -#[test] -fn core_2026_07_encoding_set_is_pinned() { - let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); - let encodings = session.encodings_in(&CORE_2026_07_0); - let ids: Vec<&str> = encodings - .iter() - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); - assert_eq!( - ids, - [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.masked", - "vortex.null", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.variant", - "vortex.zigzag", - "vortex.zstd", - ] - ); -} - #[test] fn encodings_in_editions_unions_families() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); From 355562d62516a1d3efd6caeb2694640a43acbe09 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 11:20:12 +0000 Subject: [PATCH 2/8] Record required_vortex_release in the frozen edition records Record it in a table of its own rather than beside each encoding. It is the one fact in a record that is not fixed at freeze time -- it is backfilled from compat-fixture evidence as that evidence appears -- so giving it its own table makes a backfill purely an added line. The append-only check parses both revisions of a modified record and permits exactly that transition: the table may gain entries, but everything else must be byte-identical and a release that was already recorded may never change or be dropped. Signed-off-by: "Joe Isaacs" --- .github/scripts/check_frozen_editions.py | 71 ++++++++++++++++++++---- vortex/editions/core2025.05.0.toml | 5 ++ vortex/editions/core2025.06.0.toml | 5 ++ vortex/editions/core2025.10.0.toml | 5 ++ vortex/editions/core2026.07.0.toml | 5 ++ vortex/editions/core2026.08.0.toml | 5 ++ vortex/src/editions/frozen.rs | 41 +++++++++++--- 7 files changed, 120 insertions(+), 17 deletions(-) diff --git a/.github/scripts/check_frozen_editions.py b/.github/scripts/check_frozen_editions.py index 57f4b4944a0..4a14124fa9f 100644 --- a/.github/scripts/check_frozen_editions.py +++ b/.github/scripts/check_frozen_editions.py @@ -5,6 +5,10 @@ legal edit to the directory is adding a file for a newly frozen edition, and that edition must be newer than every edition already recorded for its family. +The one exception is `required_vortex_release`, which is backfilled from compat-fixture +evidence after an edition freezes. An existing record may gain entries in that table, but +never change or lose one, and never change anything else. + Usage: python3 check_frozen_editions.py --base origin/develop """ @@ -15,7 +19,9 @@ import re import subprocess import sys +import tomllib from pathlib import Path +from typing import Any RECORD_DIR = "vortex/editions" @@ -25,12 +31,14 @@ r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" ) -EDITION_FIELD = re.compile(r'^edition = "(?P[^"]+)"$', re.MULTILINE) +# The table a frozen record may gain entries in. Everything else is fixed at freeze time. +BACKFILL_TABLE = "required_vortex_release" REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" " vortex/src/editions// and regenerate the records with\n" - " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`." + " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`.\n" + f"Only `{BACKFILL_TABLE}` may gain entries in a record that already exists." ) @@ -90,6 +98,46 @@ def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: return newest +def parse_record(text: str, path: str) -> dict[str, Any]: + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + sys.exit(f"{path} is not valid TOML: {error}") + + +def check_modification(base: str, path: str) -> list[str]: + """A modified record is legal only when it purely gains backfill entries.""" + name = Path(path).name + before = parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") + after = parse_record(Path(path).read_text(), path) + + frozen_before = {key: value for key, value in before.items() if key != BACKFILL_TABLE} + frozen_after = {key: value for key, value in after.items() if key != BACKFILL_TABLE} + if frozen_before != frozen_after: + changed = sorted( + key + for key in frozen_before.keys() | frozen_after.keys() + if frozen_before.get(key) != frozen_after.get(key) + ) + return [f"modifies the frozen record {name}: {', '.join(changed)}"] + + releases_before = before.get(BACKFILL_TABLE, {}) + releases_after = after.get(BACKFILL_TABLE, {}) + errors = [] + for encoding, release in sorted(releases_before.items()): + if encoding not in releases_after: + errors.append( + f"drops the recorded {BACKFILL_TABLE} of {encoding} from {name}", + ) + elif releases_after[encoding] != release: + errors.append( + f"changes the {BACKFILL_TABLE} of {encoding} in {name} from {release!r} " + f"to {releases_after[encoding]!r}; a recorded release is evidence and never " + "changes" + ) + return errors + + def check(base: str) -> list[str]: errors: list[str] = [] added: list[str] = [] @@ -97,9 +145,13 @@ def check(base: str) -> list[str]: for status, paths in changed_records(base): if status == "A": added.extend(paths) - continue - verb = {"M": "modifies", "D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} - errors.append(f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}") + elif status == "M": + errors.extend(check_modification(base, paths[0])) + else: + verb = {"D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} + errors.append( + f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}" + ) newest = recorded_at(base) for path in sorted(added): @@ -115,13 +167,12 @@ def check(base: str) -> list[str]: ) # The file name is the edition's identity, so it has to agree with the content. - text = Path(path).read_text() - match = EDITION_FIELD.search(text) - if match is None: + edition = parse_record(Path(path).read_text(), path).get("edition") + if edition is None: errors.append(f"adds {name}, which has no `edition` field") - elif match["edition"] != name.removesuffix(".toml"): + elif edition != name.removesuffix(".toml"): errors.append( - f"adds {name}, which records edition {match['edition']!r}; the file name " + f"adds {name}, which records edition {edition!r}; the file name " "must be the edition id" ) diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml index 0faeeb2333a..5fe30870de5 100644 --- a/vortex/editions/core2025.05.0.toml +++ b/vortex/editions/core2025.05.0.toml @@ -62,3 +62,8 @@ encodings = [ "vortex.varbinview", "vortex.zigzag", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml index 2a584ba87e6..50cc2fda9ee 100644 --- a/vortex/editions/core2025.06.0.toml +++ b/vortex/editions/core2025.06.0.toml @@ -45,3 +45,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml index 08c0688624c..94318862cfc 100644 --- a/vortex/editions/core2025.10.0.toml +++ b/vortex/editions/core2025.10.0.toml @@ -50,3 +50,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml index 53e3f4c09ae..6e73f0167e4 100644 --- a/vortex/editions/core2026.07.0.toml +++ b/vortex/editions/core2026.07.0.toml @@ -48,3 +48,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml index c1920b095cf..a8708950c8d 100644 --- a/vortex/editions/core2026.08.0.toml +++ b/vortex/editions/core2026.08.0.toml @@ -49,3 +49,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/src/editions/frozen.rs b/vortex/src/editions/frozen.rs index f538e777d6c..4972eab0121 100644 --- a/vortex/src/editions/frozen.rs +++ b/vortex/src/editions/frozen.rs @@ -18,11 +18,14 @@ //! UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen //! ``` //! -//! Only facts that are frozen by freezing are recorded. In particular -//! [`vortex_edition::EditionInclusion::required_vortex_release`] is not: it is backfilled -//! from compat-fixture evidence long after an edition freezes, and pinning it here would put -//! that legitimate backfill in conflict with the append-only rule. - +//! [`vortex_edition::EditionInclusion::required_vortex_release`] is recorded in its own +//! table rather than beside each encoding, because it is the one fact here that is not fixed +//! at freeze time: it is backfilled from compat-fixture evidence as that evidence appears. +//! Keeping it in a table of its own makes a backfill purely an added line, so the record +//! stays append-only in the literal sense and CI can allow the fill-in while still rejecting +//! a change to a release that was already recorded. + +use std::collections::BTreeMap; use std::collections::BTreeSet; use std::env; use std::fs; @@ -83,6 +86,14 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) .filter(|inclusion| inclusion.since == edition.id) .map(|inclusion| inclusion.encoding_id.as_str()) .collect(); + let releases: BTreeMap<&str, &str> = inclusions + .iter() + .filter_map(|inclusion| { + inclusion + .required_vortex_release + .map(|release| (inclusion.encoding_id.as_str(), release)) + }) + .collect(); let list = |ids: &BTreeSet<&str>| -> Vec { ids.iter().map(|id| format!(" \"{id}\",")).collect() @@ -108,7 +119,22 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) "encodings = [".to_string(), ]); lines.extend(list(&members)); - lines.extend(["]".to_string(), String::new()]); + lines.extend([ + "]".to_string(), + String::new(), + "# The earliest Vortex release able to read each encoding, recorded from evidence as" + .to_string(), + "# that evidence appears. Unlike the rest of this file it is filled in after the" + .to_string(), + "# edition freezes, so entries are only ever added, never changed.".to_string(), + "[required_vortex_release]".to_string(), + ]); + lines.extend( + releases + .iter() + .map(|(id, release)| format!("\"{id}\" = \"{release}\"")), + ); + lines.push(String::new()); lines.join("\n") } @@ -171,7 +197,8 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { "the record of frozen edition {} no longer matches its declaration.\n\ A frozen edition's encodings are fixed forever: declare a new edition \ instead of changing this one.\n\ - If the edition is genuinely new, regenerate with `{REGENERATE}`.\n\ + If you froze a new edition or recorded a required_vortex_release, \ + regenerate with `{REGENERATE}`.\n\ Record: {}", edition.id, path.display(), From 50163e10b8e388765a88f657dbdd2265eb3fe0af Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 11:30:01 +0000 Subject: [PATCH 3/8] Record every edition, and lock a record only once its edition freezes Editions are drafts until a min_vortex_version is recorded, and a draft is free to change, so recording only frozen editions left the drafts invisible. Generate a record for every declared edition instead. A draft's record may change, move, or go away with the draft; freezing turns the record into a read-forever contract that never changes again. The CI check reads frozen-ness from the record at the base revision, so a diff cannot unfreeze an edition and edit it in the same change, and the generator refuses to unfreeze a record it finds on disk. Encoding ids in a declaration may now be paired with the release that first read them -- `&("vortex.alp", "0.36.0")` -- which flows into EditionInclusion::required_vortex_release and into the records. Each core edition's members are recorded as requiring the release current when that edition froze, checked against the published release timeline: 0.36.0 (2025-05-28), 0.40.0 (2025-06-26), 0.54.0 (2025-10-20), 0.84.0 (2026-08-07), and 0.65.0 (2026-03-25) for core2026.07.0, whose only member vortex.variant first shipped there. These are upper bounds under each edition's own immutable min_vortex_version, so the check permits refining them as compat-fixture evidence narrows them, but never dropping one. Renamed the module and script from `frozen` to `records`, since they now cover drafts too. Signed-off-by: "Joe Isaacs" --- ...n_editions.py => check_edition_records.py} | 112 +++++++------ .github/workflows/ci.yml | 14 +- vortex-edition/src/lib.rs | 32 +++- vortex-edition/src/tests.rs | 45 ++++++ vortex/editions/core2025.05.0.toml | 33 +++- vortex/editions/core2025.06.0.toml | 36 ++++- vortex/editions/core2025.10.0.toml | 40 ++++- vortex/editions/core2026.07.0.toml | 41 ++++- vortex/editions/core2026.08.0.toml | 42 ++++- vortex/editions/unstable2025.05.0.toml | 24 +++ vortex/editions/unstable2026.02.0.toml | 25 +++ vortex/editions/unstable2026.04.0.toml | 36 +++++ vortex/editions/unstable2026.06.0.toml | 32 ++++ vortex/src/editions/core/v2025_05.rs | 46 +++--- vortex/src/editions/core/v2025_06.rs | 6 +- vortex/src/editions/core/v2025_10.rs | 8 +- vortex/src/editions/core/v2026_07.rs | 2 +- vortex/src/editions/core/v2026_08.rs | 2 +- vortex/src/editions/mod.rs | 2 +- vortex/src/editions/{frozen.rs => records.rs} | 148 +++++++++++------- 20 files changed, 553 insertions(+), 173 deletions(-) rename .github/scripts/{check_frozen_editions.py => check_edition_records.py} (61%) create mode 100644 vortex/editions/unstable2025.05.0.toml create mode 100644 vortex/editions/unstable2026.02.0.toml create mode 100644 vortex/editions/unstable2026.04.0.toml create mode 100644 vortex/editions/unstable2026.06.0.toml rename vortex/src/editions/{frozen.rs => records.rs} (51%) diff --git a/.github/scripts/check_frozen_editions.py b/.github/scripts/check_edition_records.py similarity index 61% rename from .github/scripts/check_frozen_editions.py rename to .github/scripts/check_edition_records.py index 4a14124fa9f..074947242fc 100644 --- a/.github/scripts/check_frozen_editions.py +++ b/.github/scripts/check_edition_records.py @@ -1,16 +1,23 @@ #!/usr/bin/env python3 -"""Check that the frozen edition records under `vortex/editions` are append-only. +"""Check that frozen edition records under `vortex/editions` never change. -A frozen edition carries a read-forever guarantee, so its record may never change: the only -legal edit to the directory is adding a file for a newly frozen edition, and that edition must -be newer than every edition already recorded for its family. +A record's mutability follows its edition. A draft is still being assembled, so its record may +change, be renamed, or be dropped. Freezing -- recording a `min_vortex_version` -- turns the +record into a read-forever contract, and from then on it may never change again. Whether a +record was frozen is read from the base revision, so a change cannot unfreeze an edition and +edit it in the same diff. -The one exception is `required_vortex_release`, which is backfilled from compat-fixture -evidence after an edition freezes. An existing record may gain entries in that table, but -never change or lose one, and never change anything else. +A newly added record must also be newer than every edition already recorded for its family: +editions are only ever added going forward. + +The one part of a frozen record that may still move is the `required_vortex_release` table. +It holds an upper bound recorded from the release current when the edition froze, refined as +compat-fixture evidence narrows it; it stays under the edition's own immutable +`min_vortex_version`, so refining it breaks no published guarantee. Entries may be added or +refined, never dropped. Usage: - python3 check_frozen_editions.py --base origin/develop + python3 check_edition_records.py --base origin/develop """ from __future__ import annotations @@ -31,14 +38,17 @@ r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" ) -# The table a frozen record may gain entries in. Everything else is fixed at freeze time. -BACKFILL_TABLE = "required_vortex_release" +# A record carries this exactly when the edition it records is frozen. +FROZEN_MARKER = "min_vortex_version" + +# The table a frozen record may still gain or refine entries in. +EVIDENCE_TABLE = "required_vortex_release" REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" " vortex/src/editions// and regenerate the records with\n" - " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`.\n" - f"Only `{BACKFILL_TABLE}` may gain entries in a record that already exists." + " `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n" + f"In a frozen record only `{EVIDENCE_TABLE}` may gain or refine entries." ) @@ -62,6 +72,17 @@ def merge_base(base: str) -> str: return result.stdout.strip() +def parse_record(text: str, path: str) -> dict[str, Any]: + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + sys.exit(f"{path} is not valid TOML: {error}") + + +def record_at(base: str, path: str) -> dict[str, Any]: + return parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") + + def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: """Split a record file name into its family and its chronological sort key.""" match = RECORD_NAME.match(name) @@ -98,44 +119,33 @@ def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: return newest -def parse_record(text: str, path: str) -> dict[str, Any]: - try: - return tomllib.loads(text) - except tomllib.TOMLDecodeError as error: - sys.exit(f"{path} is not valid TOML: {error}") - - -def check_modification(base: str, path: str) -> list[str]: - """A modified record is legal only when it purely gains backfill entries.""" +def check_modification(before: dict[str, Any], path: str) -> list[str]: + """A frozen record may only gain or refine evidence entries.""" name = Path(path).name - before = parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") after = parse_record(Path(path).read_text(), path) - frozen_before = {key: value for key, value in before.items() if key != BACKFILL_TABLE} - frozen_after = {key: value for key, value in after.items() if key != BACKFILL_TABLE} - if frozen_before != frozen_after: + fixed_before = {key: value for key, value in before.items() if key != EVIDENCE_TABLE} + fixed_after = {key: value for key, value in after.items() if key != EVIDENCE_TABLE} + if fixed_before != fixed_after: changed = sorted( key - for key in frozen_before.keys() | frozen_after.keys() - if frozen_before.get(key) != frozen_after.get(key) + for key in fixed_before.keys() | fixed_after.keys() + if fixed_before.get(key) != fixed_after.get(key) ) + if FROZEN_MARKER in changed and FROZEN_MARKER not in after: + return [ + f"unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a " + "read-forever guarantee and may never return to draft" + ] return [f"modifies the frozen record {name}: {', '.join(changed)}"] - releases_before = before.get(BACKFILL_TABLE, {}) - releases_after = after.get(BACKFILL_TABLE, {}) - errors = [] - for encoding, release in sorted(releases_before.items()): - if encoding not in releases_after: - errors.append( - f"drops the recorded {BACKFILL_TABLE} of {encoding} from {name}", - ) - elif releases_after[encoding] != release: - errors.append( - f"changes the {BACKFILL_TABLE} of {encoding} in {name} from {release!r} " - f"to {releases_after[encoding]!r}; a recorded release is evidence and never " - "changes" - ) - return errors + dropped = sorted(before.get(EVIDENCE_TABLE, {}).keys() - after.get(EVIDENCE_TABLE, {}).keys()) + if dropped: + return [ + f"drops the recorded {EVIDENCE_TABLE} of {', '.join(dropped)} from {name}; " + "recorded evidence is only ever added or refined" + ] + return [] def check(base: str) -> list[str]: @@ -144,9 +154,17 @@ def check(base: str) -> list[str]: for status, paths in changed_records(base): if status == "A": - added.extend(paths) - elif status == "M": - errors.extend(check_modification(base, paths[0])) + added.append(paths[0]) + continue + + # Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and + # then edit it. A draft's record is free to change, move, or go away with the draft. + before = record_at(base, paths[0]) + if FROZEN_MARKER not in before: + continue + + if status == "M": + errors.extend(check_modification(before, paths[0])) else: verb = {"D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} errors.append( @@ -191,10 +209,10 @@ def main() -> int: base = merge_base(args.base) errors = check(base) if not errors: - print(f"{RECORD_DIR} is append-only against {args.base} ({base[:12]}).") + print(f"{RECORD_DIR} preserves every frozen record against {args.base} ({base[:12]}).") return 0 - print(f"This change breaks the frozen edition records in {RECORD_DIR}:\n", file=sys.stderr) + print(f"This change breaks the edition records in {RECORD_DIR}:\n", file=sys.stderr) for error in errors: print(f" - it {error}", file=sys.stderr) print(f"\n{REMEDY}", file=sys.stderr) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cee16d5a4d4..8fe923b678e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,8 +64,8 @@ jobs: -c .yamllint.yaml \ .github/ - frozen-editions: - name: "Frozen editions are append-only" + edition-records: + name: "Frozen edition records never change" runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -73,10 +73,10 @@ jobs: with: # The check compares against the merge base, so it needs real history. fetch-depth: 0 - - name: Check frozen edition records + - name: Check edition records run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" - python3 .github/scripts/check_frozen_editions.py --base "$BASE" + python3 .github/scripts/check_edition_records.py --base "$BASE" python-lint: name: "Python (lint)" @@ -677,11 +677,11 @@ jobs: - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - - name: "regenerate the frozen edition records" + - name: "regenerate the edition records" env: - UPDATE_FROZEN_EDITIONS: "1" + UPDATE_EDITION_RECORDS: "1" run: | - cargo test --profile ci -p vortex --lib editions::frozen + cargo test --profile ci -p vortex --lib editions::records - name: "Make sure no files changed after regenerating" run: | git status --porcelain diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index a8eaf8e8287..42082a0ba71 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -153,9 +153,19 @@ pub struct EditionInclusion { /// Implemented for raw id strings (`"vortex.alp"`) and interned [`Id`]s here; encoding /// vtables implement it where they are defined, so a declaration can name the vtable /// (`&Primitive`) instead of spelling its id. +/// +/// Pairing an encoding with a release records the evidence that the release can read it: +/// `&("vortex.alp", "0.36.0")` declares the same membership as `&"vortex.alp"` and +/// additionally sets [`EditionInclusion::required_vortex_release`]. pub trait AsEncodingId: Debug + Send + Sync { /// The interned encoding id. fn encoding_id(&self) -> Id; + + /// The earliest Vortex release able to read and execute the encoding, when the evidence + /// has been recorded. Naming an encoding on its own leaves this unrecorded. + fn required_vortex_release(&self) -> Option<&'static str> { + None + } } impl AsEncodingId for str { @@ -182,6 +192,18 @@ impl AsEncodingId for &'static str { } } +// Pairing any encoding name with a release records the evidence alongside the membership: +// `&("vortex.alp", "0.36.0")`, or `&(&Primitive, "0.36.0")` when naming the vtable. +impl AsEncodingId for (&'static E, &'static str) { + fn encoding_id(&self) -> Id { + self.0.encoding_id() + } + + fn required_vortex_release(&self) -> Option<&'static str> { + Some(self.1) + } +} + /// Declares an edition together with the encodings that join the family at it, in one /// block. Registered with [`EditionSession::declare`], which derives each encoding's /// membership (`since` = the declared edition) from the block structure. @@ -190,18 +212,22 @@ pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, /// The encodings that join the family at this edition, named by id string or by - /// vtable. Members of earlier editions are inherited and never restated. + /// vtable, optionally paired with the release that first read them + /// (`&("vortex.alp", "0.36.0")`). Members of earlier editions are inherited and never + /// restated. pub added: &'static [&'static dyn AsEncodingId], } impl EditionInclusion { /// Declare that an encoding is a member of `since` and every later edition of the same - /// family. The encoding can be named by id string or by vtable. + /// family. The encoding can be named by id string or by vtable, and carries its + /// [`EditionInclusion::required_vortex_release`] when named as a + /// `(encoding, release)` pair. pub fn new(encoding: &E, since: EditionId) -> Self { Self { encoding_id: encoding.encoding_id(), since, - required_vortex_release: None, + required_vortex_release: encoding.required_vortex_release(), } } diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 09e1345615a..c8b9a2fe5af 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -274,3 +274,48 @@ fn edition_ids_order_within_family_only() { fn edition_id_display() { assert_eq!(FIRST.to_string(), "test2026.01.0"); } + +#[test] +fn declarations_carry_required_releases() -> Result<(), crate::EditionError> { + let editions = EditionSession::empty(); + editions.declare(&EditionDeclaration { + edition: Edition { + id: FIRST, + min_vortex_version: Some("0.40.0"), + }, + added: &[&"test.alpha", &("test.beta", "0.36.0")], + })?; + editions.validate()?; + + let inclusions = editions.encodings_in(&FIRST); + let releases: Vec<(&str, Option<&str>)> = inclusions + .iter() + .map(|inclusion| { + ( + inclusion.encoding_id.as_str(), + inclusion.required_vortex_release, + ) + }) + .collect(); + assert_eq!( + releases, + [("test.alpha", None), ("test.beta", Some("0.36.0"))] + ); + + Ok(()) +} + +#[test] +fn a_member_may_not_require_a_release_newer_than_its_edition() -> Result<(), crate::EditionError> { + let editions = EditionSession::empty(); + editions.declare(&EditionDeclaration { + edition: Edition { + id: FIRST, + min_vortex_version: Some("0.40.0"), + }, + added: &[&("test.alpha", "0.54.0")], + })?; + assert!(editions.validate().is_err()); + + Ok(()) +} diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml index 5fe30870de5..73a29ebd814 100644 --- a/vortex/editions/core2025.05.0.toml +++ b/vortex/editions/core2025.05.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2025.05.0" family = "core" @@ -63,7 +63,30 @@ encodings = [ "vortex.zigzag", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.null" = "0.36.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.zigzag" = "0.36.0" diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml index 50cc2fda9ee..0e8fbe5c1be 100644 --- a/vortex/editions/core2025.06.0.toml +++ b/vortex/editions/core2025.06.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2025.06.0" family = "core" @@ -46,7 +46,33 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml index 94318862cfc..6c6e6edebfc 100644 --- a/vortex/editions/core2025.10.0.toml +++ b/vortex/editions/core2025.10.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2025.10.0" family = "core" @@ -51,7 +51,37 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"fastlanes.rle" = "0.54.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fixed_size_list" = "0.54.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.listview" = "0.54.0" +"vortex.masked" = "0.54.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml index 6e73f0167e4..3bb6bf4331a 100644 --- a/vortex/editions/core2026.07.0.toml +++ b/vortex/editions/core2026.07.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2026.07.0" family = "core" @@ -49,7 +49,38 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"fastlanes.rle" = "0.54.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fixed_size_list" = "0.54.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.listview" = "0.54.0" +"vortex.masked" = "0.54.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.variant" = "0.65.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml index a8708950c8d..bcbfce7d1b8 100644 --- a/vortex/editions/core2026.08.0.toml +++ b/vortex/editions/core2026.08.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2026.08.0" family = "core" @@ -50,7 +50,39 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"fastlanes.rle" = "0.54.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fixed_size_list" = "0.54.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.listview" = "0.54.0" +"vortex.map" = "0.84.0" +"vortex.masked" = "0.54.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.variant" = "0.65.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/unstable2025.05.0.toml b/vortex/editions/unstable2025.05.0.toml new file mode 100644 index 00000000000..8f4ce5eb7b3 --- /dev/null +++ b/vortex/editions/unstable2025.05.0.toml @@ -0,0 +1,24 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2025.05.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.delta", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/editions/unstable2026.02.0.toml b/vortex/editions/unstable2026.02.0.toml new file mode 100644 index 00000000000..c6473da2232 --- /dev/null +++ b/vortex/editions/unstable2026.02.0.toml @@ -0,0 +1,25 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.02.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.zstd_buffers", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.zstd_buffers", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/editions/unstable2026.04.0.toml b/vortex/editions/unstable2026.04.0.toml new file mode 100644 index 00000000000..ae09e340388 --- /dev/null +++ b/vortex/editions/unstable2026.04.0.toml @@ -0,0 +1,36 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.04.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/editions/unstable2026.06.0.toml b/vortex/editions/unstable2026.06.0.toml new file mode 100644 index 00000000000..e288075ad94 --- /dev/null +++ b/vortex/editions/unstable2026.06.0.toml @@ -0,0 +1,32 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.06.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.onpair", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.onpair", + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex/src/editions/core/v2025_05.rs index a25a6f971a4..5b5e3d4b4f6 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex/src/editions/core/v2025_05.rs @@ -17,28 +17,28 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.36.0"), }, added: &[ - &"fastlanes.bitpacked", - &"fastlanes.for", - &"vortex.alp", - &"vortex.alprd", - &"vortex.bool", - &"vortex.bytebool", - &"vortex.chunked", - &"vortex.constant", - &"vortex.datetimeparts", - &"vortex.decimal", - &"vortex.decimal_byte_parts", - &"vortex.dict", - &"vortex.ext", - &"vortex.fsst", - &"vortex.list", - &"vortex.null", - &"vortex.primitive", - &"vortex.runend", - &"vortex.sparse", - &"vortex.struct", - &"vortex.varbin", - &"vortex.varbinview", - &"vortex.zigzag", + &("fastlanes.bitpacked", "0.36.0"), + &("fastlanes.for", "0.36.0"), + &("vortex.alp", "0.36.0"), + &("vortex.alprd", "0.36.0"), + &("vortex.bool", "0.36.0"), + &("vortex.bytebool", "0.36.0"), + &("vortex.chunked", "0.36.0"), + &("vortex.constant", "0.36.0"), + &("vortex.datetimeparts", "0.36.0"), + &("vortex.decimal", "0.36.0"), + &("vortex.decimal_byte_parts", "0.36.0"), + &("vortex.dict", "0.36.0"), + &("vortex.ext", "0.36.0"), + &("vortex.fsst", "0.36.0"), + &("vortex.list", "0.36.0"), + &("vortex.null", "0.36.0"), + &("vortex.primitive", "0.36.0"), + &("vortex.runend", "0.36.0"), + &("vortex.sparse", "0.36.0"), + &("vortex.struct", "0.36.0"), + &("vortex.varbin", "0.36.0"), + &("vortex.varbinview", "0.36.0"), + &("vortex.zigzag", "0.36.0"), ], }; diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex/src/editions/core/v2025_06.rs index 015325b8429..6b103df0db9 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex/src/editions/core/v2025_06.rs @@ -16,5 +16,9 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2025_06_0, min_vortex_version: Some("0.40.0"), }, - added: &[&"vortex.pco", &"vortex.sequence", &"vortex.zstd"], + added: &[ + &("vortex.pco", "0.40.0"), + &("vortex.sequence", "0.40.0"), + &("vortex.zstd", "0.40.0"), + ], }; diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex/src/editions/core/v2025_10.rs index ae21026b595..98dab5991d8 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex/src/editions/core/v2025_10.rs @@ -17,9 +17,9 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.54.0"), }, added: &[ - &"fastlanes.rle", - &"vortex.fixed_size_list", - &"vortex.listview", - &"vortex.masked", + &("fastlanes.rle", "0.54.0"), + &("vortex.fixed_size_list", "0.54.0"), + &("vortex.listview", "0.54.0"), + &("vortex.masked", "0.54.0"), ], }; diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex/src/editions/core/v2026_07.rs index 820c68cf7dd..61b362b16cc 100644 --- a/vortex/src/editions/core/v2026_07.rs +++ b/vortex/src/editions/core/v2026_07.rs @@ -16,5 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2026_07_0, min_vortex_version: Some("0.65.0"), }, - added: &[&"vortex.variant"], + added: &[&("vortex.variant", "0.65.0")], }; diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index 4d47cfbe027..5e89b6875c3 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -16,5 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2026_08, min_vortex_version: Some("0.84.0"), }, - added: &[&"vortex.map"], + added: &[&("vortex.map", "0.84.0")], }; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index ab20470f2f7..41a1f27854b 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -15,7 +15,7 @@ pub mod core; #[cfg(test)] -mod frozen; +mod records; #[cfg(test)] mod tests; pub mod unstable; diff --git a/vortex/src/editions/frozen.rs b/vortex/src/editions/records.rs similarity index 51% rename from vortex/src/editions/frozen.rs rename to vortex/src/editions/records.rs index 4972eab0121..0c42a81b285 100644 --- a/vortex/src/editions/frozen.rs +++ b/vortex/src/editions/records.rs @@ -1,34 +1,37 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The frozen-edition record under `vortex/editions`. +//! The edition records under `vortex/editions`. //! -//! A frozen edition carries a read-forever guarantee, so its encoding set must never change -//! again. Every frozen edition has one generated TOML file recording that contract: the -//! identifier, the minimum Vortex version whose reader supports it, and the full encoding -//! set. Freezing a new edition adds a file; nothing else may touch the directory, which CI -//! enforces by rejecting any diff that modifies or deletes an existing record -//! (`.github/scripts/check_frozen_editions.py`). +//! Every declared edition has one generated TOML file recording what it contains: the +//! identifier, the minimum Vortex version whose reader supports it once frozen, the full +//! encoding set, and the release recorded for each member. //! -//! The test here keeps the record honest in the other direction: the files must match what -//! [`super::EDITION_DECLARATIONS`] actually computes, so a frozen edition cannot drift -//! without the record drifting with it. Regenerate after freezing a new edition with: +//! A record's mutability follows its edition. A draft is still being assembled, so its +//! record may change however the draft does. Freezing — recording a +//! [`vortex_edition::Edition::min_vortex_version`] — turns the record into a contract that +//! carries a read-forever guarantee, and from then on it may never change again. CI enforces +//! that by rejecting any diff that touches a record that was already frozen at the base +//! revision (`.github/scripts/check_edition_records.py`). +//! +//! The test here keeps the records honest in the other direction: they must match what +//! [`super::EDITION_DECLARATIONS`] actually computes, so an edition cannot drift without its +//! record drifting with it. Regenerate with: //! //! ```bash -//! UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen +//! UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records //! ``` //! -//! [`vortex_edition::EditionInclusion::required_vortex_release`] is recorded in its own -//! table rather than beside each encoding, because it is the one fact here that is not fixed -//! at freeze time: it is backfilled from compat-fixture evidence as that evidence appears. -//! Keeping it in a table of its own makes a backfill purely an added line, so the record -//! stays append-only in the literal sense and CI can allow the fill-in while still rejecting -//! a change to a release that was already recorded. +//! [`vortex_edition::EditionInclusion::required_vortex_release`] sits in a table of its own +//! because it is the one part of a frozen record that is still allowed to move: it is an +//! upper bound recorded from the release current when the edition froze, refined as +//! compat-fixture evidence narrows it. Entries may be added or refined, never dropped. use std::collections::BTreeMap; use std::collections::BTreeSet; use std::env; use std::fs; +use std::path::Path; use std::path::PathBuf; use anyhow::Context; @@ -39,17 +42,23 @@ use vortex_edition::EditionSession; use super::EDITION_DECLARATIONS; -/// Set to any value to rewrite the record instead of verifying it. -const UPDATE_VAR: &str = "UPDATE_FROZEN_EDITIONS"; +/// Set to any value to rewrite the records instead of verifying them. +const UPDATE_VAR: &str = "UPDATE_EDITION_RECORDS"; + +const REGENERATE: &str = "UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records"; -const REGENERATE: &str = "UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen"; +const GENERATED_BY: &str = + "# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n#"; -const HEADER: &str = "\ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. -# +const FROZEN_NOTE: &str = "\ # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI."; +# editing or deleting a frozen one is rejected by CI."; + +const DRAFT_NOTE: &str = "\ +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again."; fn record_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("editions") @@ -63,19 +72,9 @@ fn session() -> Result { Ok(session) } -/// The frozen editions, paired with the version that freezing recorded. Drafts have no -/// record: a file appears in the directory at the moment an edition freezes. -fn frozen(session: &EditionSession) -> Vec<(Edition, &'static str)> { - session - .editions() - .into_iter() - .filter_map(|edition| edition.min_vortex_version.map(|version| (edition, version))) - .collect() -} - -/// Render one edition's record. Deterministic: both encoding lists are sorted by id, so the +/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the /// generated bytes depend only on the declarations. -fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) -> String { +fn record(session: &EditionSession, edition: &Edition) -> String { let inclusions = session.encodings_in(&edition.id); let members: BTreeSet<&str> = inclusions .iter() @@ -99,16 +98,26 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) ids.iter().map(|id| format!(" \"{id}\",")).collect() }; + let note = if edition.is_draft() { + DRAFT_NOTE + } else { + FROZEN_NOTE + }; let mut lines = vec![ - HEADER.to_string(), + GENERATED_BY.to_string(), + note.to_string(), String::new(), format!("edition = \"{}\"", edition.id), format!("family = \"{}\"", edition.id.family), - format!("min_vortex_version = \"{min_vortex_version}\""), + ]; + if let Some(min_vortex_version) = edition.min_vortex_version { + lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); + } + lines.extend([ String::new(), "# The encodings that join the family at this edition.".to_string(), "added = [".to_string(), - ]; + ]); lines.extend(list(&added)); lines.extend([ "]".to_string(), @@ -122,11 +131,11 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) lines.extend([ "]".to_string(), String::new(), - "# The earliest Vortex release able to read each encoding, recorded from evidence as" + "# The earliest Vortex release able to read each encoding. Recorded as an upper bound" .to_string(), - "# that evidence appears. Unlike the rest of this file it is filled in after the" + "# from the release current when the edition froze, and refined as compat-fixture" .to_string(), - "# edition freezes, so entries are only ever added, never changed.".to_string(), + "# evidence narrows it, so entries are added or refined but never dropped.".to_string(), "[required_vortex_release]".to_string(), ]); lines.extend( @@ -138,12 +147,12 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) lines.join("\n") } -fn record_path(dir: &std::path::Path, edition: &Edition) -> PathBuf { +fn record_path(dir: &Path, edition: &Edition) -> PathBuf { dir.join(format!("{}.toml", edition.id)) } /// The `*.toml` file names present in the record directory. -fn existing_records(dir: &std::path::Path) -> anyhow::Result> { +fn existing_records(dir: &Path) -> anyhow::Result> { let mut names = BTreeSet::new(); for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { let path = entry?.path(); @@ -158,14 +167,21 @@ fn existing_records(dir: &std::path::Path) -> anyhow::Result> { Ok(names) } -/// Every frozen edition has a record, every record matches the declarations exactly, and no -/// record exists without a frozen edition behind it. +/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. +fn records_a_frozen_edition(contents: &str) -> bool { + contents + .lines() + .any(|line| line.starts_with("min_vortex_version = ")) +} + +/// Every declared edition has a record, every record matches the declarations exactly, no +/// record exists without a declaration behind it, and no frozen edition is returned to draft. /// -/// The third check is what catches a frozen edition being deleted or unfrozen, so the update -/// mode deliberately never removes a file: unfreezing cannot be laundered through the -/// generator. +/// The last two are what catch an edition being deleted or unfrozen, so the update mode +/// deliberately never removes a file and never unfreezes one: neither can be laundered +/// through the generator. #[test] -fn records_match_the_frozen_editions() -> anyhow::Result<()> { +fn records_match_the_declared_editions() -> anyhow::Result<()> { let session = session()?; let dir = record_dir(); let update = env::var_os(UPDATE_VAR).is_some(); @@ -173,9 +189,9 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; let mut expected_names = BTreeSet::new(); - for (edition, min_vortex_version) in frozen(&session) { + for edition in session.editions() { let path = record_path(&dir, &edition); - let expected = record(&session, &edition, min_vortex_version); + let expected = record(&session, &edition); expected_names.insert(format!("{}.toml", edition.id)); let actual = match fs::read_to_string(&path) { @@ -184,6 +200,18 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), }; + if let Some(actual) = &actual + && edition.is_draft() + && records_a_frozen_edition(actual) + { + return Err(anyhow!( + "{} is recorded as frozen but its declaration is now a draft.\n\ + Freezing is permanent: an edition that has recorded a min_vortex_version \ + carries a read-forever guarantee and may never return to draft.", + edition.id, + )); + } + if actual.as_deref() == Some(expected.as_str()) { continue; } @@ -194,17 +222,17 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { return Err(match actual { Some(_) => anyhow!( - "the record of frozen edition {} no longer matches its declaration.\n\ - A frozen edition's encodings are fixed forever: declare a new edition \ + "the record of edition {} no longer matches its declaration.\n\ + If {} is frozen its encodings are fixed forever: declare a new edition \ instead of changing this one.\n\ - If you froze a new edition or recorded a required_vortex_release, \ - regenerate with `{REGENERATE}`.\n\ + Otherwise regenerate with `{REGENERATE}`.\n\ Record: {}", edition.id, + edition.id, path.display(), ), None => anyhow!( - "frozen edition {} has no record. Regenerate with `{REGENERATE}`.\n\ + "edition {} has no record. Regenerate with `{REGENERATE}`.\n\ Expected: {}", edition.id, path.display(), @@ -218,9 +246,9 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { .collect(); if !strays.is_empty() { return Err(anyhow!( - "{} has records with no frozen edition behind them: {strays:?}.\n\ - A frozen edition may never be deleted or returned to draft; its declaration must \ - stay in `EDITION_DECLARATIONS` with its `min_vortex_version` recorded.", + "{} has records with no declared edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted; its declaration must stay in \ + `EDITION_DECLARATIONS`.", dir.display(), )); } From c1dd9b340663d7138b44dd1585abaacf91265956 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 13:32:43 +0000 Subject: [PATCH 4/8] Export the edition records from an xtask instead of a test The exporter belongs with the repo's other generated files, so move it to `cargo run -p xtask -- generate-editions`, alongside generate-fbs and generate-proto, and run it from the same generated-files CI job that already checks git is clean afterwards. The `#[cfg(test)]` module and its UPDATE_EDITION_RECORDS environment variable are gone; the two rules git history cannot see -- a record may not be deleted, and a frozen edition may not return to draft -- now fail the exporter itself. Drop the per-encoding release from the records. An edition declares the release it froze in, so repeating it on all 23 of core2025.05.0's members said nothing that the edition did not already say. Declarations return to plain encoding lists, EditionInclusion::required_vortex_release goes back to being unset until there is per-encoding evidence to record, and the append-only check simplifies to rejecting any change at all to a frozen record. Signed-off-by: "Joe Isaacs" --- .github/scripts/check_edition_records.py | 43 +--- .github/workflows/ci.yml | 8 +- Cargo.lock | 1 + vortex-edition/src/lib.rs | 32 +-- vortex-edition/src/tests.rs | 45 ---- vortex/editions/core2025.05.0.toml | 30 +-- vortex/editions/core2025.06.0.toml | 33 +-- vortex/editions/core2025.10.0.toml | 37 +--- vortex/editions/core2026.07.0.toml | 38 +--- vortex/editions/core2026.08.0.toml | 39 +--- vortex/editions/unstable2025.05.0.toml | 7 +- vortex/editions/unstable2026.02.0.toml | 7 +- vortex/editions/unstable2026.04.0.toml | 7 +- vortex/editions/unstable2026.06.0.toml | 7 +- vortex/src/editions/core/v2025_05.rs | 46 ++-- vortex/src/editions/core/v2025_06.rs | 6 +- vortex/src/editions/core/v2025_10.rs | 8 +- vortex/src/editions/core/v2026_07.rs | 2 +- vortex/src/editions/core/v2026_08.rs | 2 +- vortex/src/editions/mod.rs | 2 - vortex/src/editions/records.rs | 257 ----------------------- xtask/Cargo.toml | 1 + xtask/src/generate_editions.rs | 165 +++++++++++++++ xtask/src/main.rs | 14 +- 24 files changed, 233 insertions(+), 604 deletions(-) delete mode 100644 vortex/src/editions/records.rs create mode 100644 xtask/src/generate_editions.rs diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index 074947242fc..9a3348e8b07 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -10,12 +10,6 @@ A newly added record must also be newer than every edition already recorded for its family: editions are only ever added going forward. -The one part of a frozen record that may still move is the `required_vortex_release` table. -It holds an upper bound recorded from the release current when the edition froze, refined as -compat-fixture evidence narrows it; it stays under the edition's own immutable -`min_vortex_version`, so refining it breaks no published guarantee. Entries may be added or -refined, never dropped. - Usage: python3 check_edition_records.py --base origin/develop """ @@ -41,14 +35,10 @@ # A record carries this exactly when the edition it records is frozen. FROZEN_MARKER = "min_vortex_version" -# The table a frozen record may still gain or refine entries in. -EVIDENCE_TABLE = "required_vortex_release" - REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" " vortex/src/editions// and regenerate the records with\n" - " `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n" - f"In a frozen record only `{EVIDENCE_TABLE}` may gain or refine entries." + " `cargo run -p xtask -- generate-editions`." ) @@ -120,32 +110,21 @@ def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: def check_modification(before: dict[str, Any], path: str) -> list[str]: - """A frozen record may only gain or refine evidence entries.""" + """A frozen record may not change at all; name the fields that did.""" name = Path(path).name after = parse_record(Path(path).read_text(), path) - fixed_before = {key: value for key, value in before.items() if key != EVIDENCE_TABLE} - fixed_after = {key: value for key, value in after.items() if key != EVIDENCE_TABLE} - if fixed_before != fixed_after: - changed = sorted( - key - for key in fixed_before.keys() | fixed_after.keys() - if fixed_before.get(key) != fixed_after.get(key) - ) - if FROZEN_MARKER in changed and FROZEN_MARKER not in after: - return [ - f"unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a " - "read-forever guarantee and may never return to draft" - ] - return [f"modifies the frozen record {name}: {', '.join(changed)}"] - - dropped = sorted(before.get(EVIDENCE_TABLE, {}).keys() - after.get(EVIDENCE_TABLE, {}).keys()) - if dropped: + changed = sorted( + key for key in before.keys() | after.keys() if before.get(key) != after.get(key) + ) + if not changed: + return [] + if FROZEN_MARKER in changed and FROZEN_MARKER not in after: return [ - f"drops the recorded {EVIDENCE_TABLE} of {', '.join(dropped)} from {name}; " - "recorded evidence is only ever added or refined" + f"unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a " + "read-forever guarantee and may never return to draft" ] - return [] + return [f"modifies the frozen record {name}: {', '.join(changed)}"] def check(base: str) -> list[str]: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fe923b678e..ff41d03569f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -674,14 +674,12 @@ jobs: run: | cargo run --profile ci -p xtask -- generate-fbs cargo run --profile ci -p xtask -- generate-proto + - name: "regenerate the edition records" + run: | + cargo run --profile ci -p xtask -- generate-editions - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - - name: "regenerate the edition records" - env: - UPDATE_EDITION_RECORDS: "1" - run: | - cargo test --profile ci -p vortex --lib editions::records - name: "Make sure no files changed after regenerating" run: | git status --porcelain diff --git a/Cargo.lock b/Cargo.lock index c05aae6923e..bdfab76e551 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11269,6 +11269,7 @@ dependencies = [ "anyhow", "clap", "prost-build", + "vortex", "xshell", ] diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 42082a0ba71..a8eaf8e8287 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -153,19 +153,9 @@ pub struct EditionInclusion { /// Implemented for raw id strings (`"vortex.alp"`) and interned [`Id`]s here; encoding /// vtables implement it where they are defined, so a declaration can name the vtable /// (`&Primitive`) instead of spelling its id. -/// -/// Pairing an encoding with a release records the evidence that the release can read it: -/// `&("vortex.alp", "0.36.0")` declares the same membership as `&"vortex.alp"` and -/// additionally sets [`EditionInclusion::required_vortex_release`]. pub trait AsEncodingId: Debug + Send + Sync { /// The interned encoding id. fn encoding_id(&self) -> Id; - - /// The earliest Vortex release able to read and execute the encoding, when the evidence - /// has been recorded. Naming an encoding on its own leaves this unrecorded. - fn required_vortex_release(&self) -> Option<&'static str> { - None - } } impl AsEncodingId for str { @@ -192,18 +182,6 @@ impl AsEncodingId for &'static str { } } -// Pairing any encoding name with a release records the evidence alongside the membership: -// `&("vortex.alp", "0.36.0")`, or `&(&Primitive, "0.36.0")` when naming the vtable. -impl AsEncodingId for (&'static E, &'static str) { - fn encoding_id(&self) -> Id { - self.0.encoding_id() - } - - fn required_vortex_release(&self) -> Option<&'static str> { - Some(self.1) - } -} - /// Declares an edition together with the encodings that join the family at it, in one /// block. Registered with [`EditionSession::declare`], which derives each encoding's /// membership (`since` = the declared edition) from the block structure. @@ -212,22 +190,18 @@ pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, /// The encodings that join the family at this edition, named by id string or by - /// vtable, optionally paired with the release that first read them - /// (`&("vortex.alp", "0.36.0")`). Members of earlier editions are inherited and never - /// restated. + /// vtable. Members of earlier editions are inherited and never restated. pub added: &'static [&'static dyn AsEncodingId], } impl EditionInclusion { /// Declare that an encoding is a member of `since` and every later edition of the same - /// family. The encoding can be named by id string or by vtable, and carries its - /// [`EditionInclusion::required_vortex_release`] when named as a - /// `(encoding, release)` pair. + /// family. The encoding can be named by id string or by vtable. pub fn new(encoding: &E, since: EditionId) -> Self { Self { encoding_id: encoding.encoding_id(), since, - required_vortex_release: encoding.required_vortex_release(), + required_vortex_release: None, } } diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index c8b9a2fe5af..09e1345615a 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -274,48 +274,3 @@ fn edition_ids_order_within_family_only() { fn edition_id_display() { assert_eq!(FIRST.to_string(), "test2026.01.0"); } - -#[test] -fn declarations_carry_required_releases() -> Result<(), crate::EditionError> { - let editions = EditionSession::empty(); - editions.declare(&EditionDeclaration { - edition: Edition { - id: FIRST, - min_vortex_version: Some("0.40.0"), - }, - added: &[&"test.alpha", &("test.beta", "0.36.0")], - })?; - editions.validate()?; - - let inclusions = editions.encodings_in(&FIRST); - let releases: Vec<(&str, Option<&str>)> = inclusions - .iter() - .map(|inclusion| { - ( - inclusion.encoding_id.as_str(), - inclusion.required_vortex_release, - ) - }) - .collect(); - assert_eq!( - releases, - [("test.alpha", None), ("test.beta", Some("0.36.0"))] - ); - - Ok(()) -} - -#[test] -fn a_member_may_not_require_a_release_newer_than_its_edition() -> Result<(), crate::EditionError> { - let editions = EditionSession::empty(); - editions.declare(&EditionDeclaration { - edition: Edition { - id: FIRST, - min_vortex_version: Some("0.40.0"), - }, - added: &[&("test.alpha", "0.54.0")], - })?; - assert!(editions.validate().is_err()); - - Ok(()) -} diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml index 73a29ebd814..eac393d6ddf 100644 --- a/vortex/editions/core2025.05.0.toml +++ b/vortex/editions/core2025.05.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -62,31 +62,3 @@ encodings = [ "vortex.varbinview", "vortex.zigzag", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.null" = "0.36.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.zigzag" = "0.36.0" diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml index 0e8fbe5c1be..a4e63311f4c 100644 --- a/vortex/editions/core2025.06.0.toml +++ b/vortex/editions/core2025.06.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -45,34 +45,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml index 6c6e6edebfc..4e3bbba2d69 100644 --- a/vortex/editions/core2025.10.0.toml +++ b/vortex/editions/core2025.10.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -50,38 +50,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"fastlanes.rle" = "0.54.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fixed_size_list" = "0.54.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.listview" = "0.54.0" -"vortex.masked" = "0.54.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml index 3bb6bf4331a..e49b6f0bf26 100644 --- a/vortex/editions/core2026.07.0.toml +++ b/vortex/editions/core2026.07.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -48,39 +48,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"fastlanes.rle" = "0.54.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fixed_size_list" = "0.54.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.listview" = "0.54.0" -"vortex.masked" = "0.54.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.variant" = "0.65.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml index bcbfce7d1b8..82db60f548d 100644 --- a/vortex/editions/core2026.08.0.toml +++ b/vortex/editions/core2026.08.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -49,40 +49,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"fastlanes.rle" = "0.54.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fixed_size_list" = "0.54.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.listview" = "0.54.0" -"vortex.map" = "0.84.0" -"vortex.masked" = "0.54.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.variant" = "0.65.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/unstable2025.05.0.toml b/vortex/editions/unstable2025.05.0.toml index 8f4ce5eb7b3..b2b2ddb8325 100644 --- a/vortex/editions/unstable2025.05.0.toml +++ b/vortex/editions/unstable2025.05.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -17,8 +17,3 @@ added = [ encodings = [ "fastlanes.delta", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/editions/unstable2026.02.0.toml b/vortex/editions/unstable2026.02.0.toml index c6473da2232..00f1b0bef6f 100644 --- a/vortex/editions/unstable2026.02.0.toml +++ b/vortex/editions/unstable2026.02.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -18,8 +18,3 @@ encodings = [ "fastlanes.delta", "vortex.zstd_buffers", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/editions/unstable2026.04.0.toml b/vortex/editions/unstable2026.04.0.toml index ae09e340388..7e721892046 100644 --- a/vortex/editions/unstable2026.04.0.toml +++ b/vortex/editions/unstable2026.04.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -29,8 +29,3 @@ encodings = [ "vortex.tensor.normalized", "vortex.zstd_buffers", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/editions/unstable2026.06.0.toml b/vortex/editions/unstable2026.06.0.toml index e288075ad94..c890c13627d 100644 --- a/vortex/editions/unstable2026.06.0.toml +++ b/vortex/editions/unstable2026.06.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -25,8 +25,3 @@ encodings = [ "vortex.tensor.normalized", "vortex.zstd_buffers", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex/src/editions/core/v2025_05.rs index 5b5e3d4b4f6..a25a6f971a4 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex/src/editions/core/v2025_05.rs @@ -17,28 +17,28 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.36.0"), }, added: &[ - &("fastlanes.bitpacked", "0.36.0"), - &("fastlanes.for", "0.36.0"), - &("vortex.alp", "0.36.0"), - &("vortex.alprd", "0.36.0"), - &("vortex.bool", "0.36.0"), - &("vortex.bytebool", "0.36.0"), - &("vortex.chunked", "0.36.0"), - &("vortex.constant", "0.36.0"), - &("vortex.datetimeparts", "0.36.0"), - &("vortex.decimal", "0.36.0"), - &("vortex.decimal_byte_parts", "0.36.0"), - &("vortex.dict", "0.36.0"), - &("vortex.ext", "0.36.0"), - &("vortex.fsst", "0.36.0"), - &("vortex.list", "0.36.0"), - &("vortex.null", "0.36.0"), - &("vortex.primitive", "0.36.0"), - &("vortex.runend", "0.36.0"), - &("vortex.sparse", "0.36.0"), - &("vortex.struct", "0.36.0"), - &("vortex.varbin", "0.36.0"), - &("vortex.varbinview", "0.36.0"), - &("vortex.zigzag", "0.36.0"), + &"fastlanes.bitpacked", + &"fastlanes.for", + &"vortex.alp", + &"vortex.alprd", + &"vortex.bool", + &"vortex.bytebool", + &"vortex.chunked", + &"vortex.constant", + &"vortex.datetimeparts", + &"vortex.decimal", + &"vortex.decimal_byte_parts", + &"vortex.dict", + &"vortex.ext", + &"vortex.fsst", + &"vortex.list", + &"vortex.null", + &"vortex.primitive", + &"vortex.runend", + &"vortex.sparse", + &"vortex.struct", + &"vortex.varbin", + &"vortex.varbinview", + &"vortex.zigzag", ], }; diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex/src/editions/core/v2025_06.rs index 6b103df0db9..015325b8429 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex/src/editions/core/v2025_06.rs @@ -16,9 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2025_06_0, min_vortex_version: Some("0.40.0"), }, - added: &[ - &("vortex.pco", "0.40.0"), - &("vortex.sequence", "0.40.0"), - &("vortex.zstd", "0.40.0"), - ], + added: &[&"vortex.pco", &"vortex.sequence", &"vortex.zstd"], }; diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex/src/editions/core/v2025_10.rs index 98dab5991d8..ae21026b595 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex/src/editions/core/v2025_10.rs @@ -17,9 +17,9 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.54.0"), }, added: &[ - &("fastlanes.rle", "0.54.0"), - &("vortex.fixed_size_list", "0.54.0"), - &("vortex.listview", "0.54.0"), - &("vortex.masked", "0.54.0"), + &"fastlanes.rle", + &"vortex.fixed_size_list", + &"vortex.listview", + &"vortex.masked", ], }; diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex/src/editions/core/v2026_07.rs index 61b362b16cc..820c68cf7dd 100644 --- a/vortex/src/editions/core/v2026_07.rs +++ b/vortex/src/editions/core/v2026_07.rs @@ -16,5 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2026_07_0, min_vortex_version: Some("0.65.0"), }, - added: &[&("vortex.variant", "0.65.0")], + added: &[&"vortex.variant"], }; diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index 5e89b6875c3..4d47cfbe027 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -16,5 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2026_08, min_vortex_version: Some("0.84.0"), }, - added: &[&("vortex.map", "0.84.0")], + added: &[&"vortex.map"], }; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 41a1f27854b..88e4f351d0d 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -15,8 +15,6 @@ pub mod core; #[cfg(test)] -mod records; -#[cfg(test)] mod tests; pub mod unstable; diff --git a/vortex/src/editions/records.rs b/vortex/src/editions/records.rs deleted file mode 100644 index 0c42a81b285..00000000000 --- a/vortex/src/editions/records.rs +++ /dev/null @@ -1,257 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The edition records under `vortex/editions`. -//! -//! Every declared edition has one generated TOML file recording what it contains: the -//! identifier, the minimum Vortex version whose reader supports it once frozen, the full -//! encoding set, and the release recorded for each member. -//! -//! A record's mutability follows its edition. A draft is still being assembled, so its -//! record may change however the draft does. Freezing — recording a -//! [`vortex_edition::Edition::min_vortex_version`] — turns the record into a contract that -//! carries a read-forever guarantee, and from then on it may never change again. CI enforces -//! that by rejecting any diff that touches a record that was already frozen at the base -//! revision (`.github/scripts/check_edition_records.py`). -//! -//! The test here keeps the records honest in the other direction: they must match what -//! [`super::EDITION_DECLARATIONS`] actually computes, so an edition cannot drift without its -//! record drifting with it. Regenerate with: -//! -//! ```bash -//! UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records -//! ``` -//! -//! [`vortex_edition::EditionInclusion::required_vortex_release`] sits in a table of its own -//! because it is the one part of a frozen record that is still allowed to move: it is an -//! upper bound recorded from the release current when the edition froze, refined as -//! compat-fixture evidence narrows it. Entries may be added or refined, never dropped. - -use std::collections::BTreeMap; -use std::collections::BTreeSet; -use std::env; -use std::fs; -use std::path::Path; -use std::path::PathBuf; - -use anyhow::Context; -use anyhow::anyhow; -use vortex_edition::Edition; -use vortex_edition::EditionError; -use vortex_edition::EditionSession; - -use super::EDITION_DECLARATIONS; - -/// Set to any value to rewrite the records instead of verifying them. -const UPDATE_VAR: &str = "UPDATE_EDITION_RECORDS"; - -const REGENERATE: &str = "UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records"; - -const GENERATED_BY: &str = - "# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n#"; - -const FROZEN_NOTE: &str = "\ -# This edition is frozen: it carries a read-forever guarantee, so this record of what it -# contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting a frozen one is rejected by CI."; - -const DRAFT_NOTE: &str = "\ -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again."; - -fn record_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("editions") -} - -fn session() -> Result { - let session = EditionSession::empty(); - for declaration in EDITION_DECLARATIONS { - session.declare(declaration)?; - } - Ok(session) -} - -/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the -/// generated bytes depend only on the declarations. -fn record(session: &EditionSession, edition: &Edition) -> String { - let inclusions = session.encodings_in(&edition.id); - let members: BTreeSet<&str> = inclusions - .iter() - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); - let added: BTreeSet<&str> = inclusions - .iter() - .filter(|inclusion| inclusion.since == edition.id) - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); - let releases: BTreeMap<&str, &str> = inclusions - .iter() - .filter_map(|inclusion| { - inclusion - .required_vortex_release - .map(|release| (inclusion.encoding_id.as_str(), release)) - }) - .collect(); - - let list = |ids: &BTreeSet<&str>| -> Vec { - ids.iter().map(|id| format!(" \"{id}\",")).collect() - }; - - let note = if edition.is_draft() { - DRAFT_NOTE - } else { - FROZEN_NOTE - }; - let mut lines = vec![ - GENERATED_BY.to_string(), - note.to_string(), - String::new(), - format!("edition = \"{}\"", edition.id), - format!("family = \"{}\"", edition.id.family), - ]; - if let Some(min_vortex_version) = edition.min_vortex_version { - lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); - } - lines.extend([ - String::new(), - "# The encodings that join the family at this edition.".to_string(), - "added = [".to_string(), - ]); - lines.extend(list(&added)); - lines.extend([ - "]".to_string(), - String::new(), - "# The edition's full membership: the encodings above, plus every member of earlier" - .to_string(), - "# editions of the family.".to_string(), - "encodings = [".to_string(), - ]); - lines.extend(list(&members)); - lines.extend([ - "]".to_string(), - String::new(), - "# The earliest Vortex release able to read each encoding. Recorded as an upper bound" - .to_string(), - "# from the release current when the edition froze, and refined as compat-fixture" - .to_string(), - "# evidence narrows it, so entries are added or refined but never dropped.".to_string(), - "[required_vortex_release]".to_string(), - ]); - lines.extend( - releases - .iter() - .map(|(id, release)| format!("\"{id}\" = \"{release}\"")), - ); - lines.push(String::new()); - lines.join("\n") -} - -fn record_path(dir: &Path, edition: &Edition) -> PathBuf { - dir.join(format!("{}.toml", edition.id)) -} - -/// The `*.toml` file names present in the record directory. -fn existing_records(dir: &Path) -> anyhow::Result> { - let mut names = BTreeSet::new(); - for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { - let path = entry?.path(); - if path - .extension() - .is_some_and(|extension| extension == "toml") - && let Some(name) = path.file_name().and_then(|name| name.to_str()) - { - names.insert(name.to_string()); - } - } - Ok(names) -} - -/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. -fn records_a_frozen_edition(contents: &str) -> bool { - contents - .lines() - .any(|line| line.starts_with("min_vortex_version = ")) -} - -/// Every declared edition has a record, every record matches the declarations exactly, no -/// record exists without a declaration behind it, and no frozen edition is returned to draft. -/// -/// The last two are what catch an edition being deleted or unfrozen, so the update mode -/// deliberately never removes a file and never unfreezes one: neither can be laundered -/// through the generator. -#[test] -fn records_match_the_declared_editions() -> anyhow::Result<()> { - let session = session()?; - let dir = record_dir(); - let update = env::var_os(UPDATE_VAR).is_some(); - - fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; - - let mut expected_names = BTreeSet::new(); - for edition in session.editions() { - let path = record_path(&dir, &edition); - let expected = record(&session, &edition); - expected_names.insert(format!("{}.toml", edition.id)); - - let actual = match fs::read_to_string(&path) { - Ok(actual) => Some(actual), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), - }; - - if let Some(actual) = &actual - && edition.is_draft() - && records_a_frozen_edition(actual) - { - return Err(anyhow!( - "{} is recorded as frozen but its declaration is now a draft.\n\ - Freezing is permanent: an edition that has recorded a min_vortex_version \ - carries a read-forever guarantee and may never return to draft.", - edition.id, - )); - } - - if actual.as_deref() == Some(expected.as_str()) { - continue; - } - if update { - fs::write(&path, &expected).with_context(|| format!("writing {}", path.display()))?; - continue; - } - - return Err(match actual { - Some(_) => anyhow!( - "the record of edition {} no longer matches its declaration.\n\ - If {} is frozen its encodings are fixed forever: declare a new edition \ - instead of changing this one.\n\ - Otherwise regenerate with `{REGENERATE}`.\n\ - Record: {}", - edition.id, - edition.id, - path.display(), - ), - None => anyhow!( - "edition {} has no record. Regenerate with `{REGENERATE}`.\n\ - Expected: {}", - edition.id, - path.display(), - ), - }); - } - - let strays: Vec = existing_records(&dir)? - .difference(&expected_names) - .cloned() - .collect(); - if !strays.is_empty() { - return Err(anyhow!( - "{} has records with no declared edition behind them: {strays:?}.\n\ - A frozen edition may never be deleted; its declaration must stay in \ - `EDITION_DECLARATIONS`.", - dir.display(), - )); - } - - Ok(()) -} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index eae2db43413..9878d2f6d1b 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,6 +23,7 @@ test = false anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } prost-build = { workspace = true } +vortex = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs new file mode 100644 index 00000000000..961c4ba0a12 --- /dev/null +++ b/xtask/src/generate_editions.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Export the edition records under `vortex/editions`. +//! +//! Every declared edition gets one TOML file recording what it contains: the identifier, the +//! minimum Vortex version whose reader supports it once frozen, and its full encoding set. +//! +//! A record's mutability follows its edition. A draft is still being assembled, so its record +//! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a +//! contract carrying a read-forever guarantee, and from then on it may never change again. CI +//! enforces that against git history in `.github/scripts/check_edition_records.py`; this +//! exporter enforces the two rules that history cannot see, refusing to delete a record or to +//! unfreeze one. + +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::anyhow; +use vortex::editions::EDITION_DECLARATIONS; +use vortex::editions::Edition; +use vortex::editions::EditionSession; + +const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; + +const FROZEN_NOTE: &str = "\ +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI."; + +const DRAFT_NOTE: &str = "\ +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again."; + +/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the +/// generated bytes depend only on the declarations. +fn record(session: &EditionSession, edition: &Edition) -> String { + let inclusions = session.encodings_in(&edition.id); + let members: BTreeSet<&str> = inclusions + .iter() + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + let added: BTreeSet<&str> = inclusions + .iter() + .filter(|inclusion| inclusion.since == edition.id) + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + + let list = |ids: &BTreeSet<&str>| -> Vec { + ids.iter().map(|id| format!(" \"{id}\",")).collect() + }; + + let note = if edition.is_draft() { + DRAFT_NOTE + } else { + FROZEN_NOTE + }; + let mut lines = vec![ + GENERATED_BY.to_string(), + note.to_string(), + String::new(), + format!("edition = \"{}\"", edition.id), + format!("family = \"{}\"", edition.id.family), + ]; + if let Some(min_vortex_version) = edition.min_vortex_version { + lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); + } + lines.extend([ + String::new(), + "# The encodings that join the family at this edition.".to_string(), + "added = [".to_string(), + ]); + lines.extend(list(&added)); + lines.extend([ + "]".to_string(), + String::new(), + "# The edition's full membership: the encodings above, plus every member of earlier" + .to_string(), + "# editions of the family.".to_string(), + "encodings = [".to_string(), + ]); + lines.extend(list(&members)); + lines.extend(["]".to_string(), String::new()]); + lines.join("\n") +} + +/// The `*.toml` file names present in the record directory. +fn existing_records(dir: &Path) -> anyhow::Result> { + let mut names = BTreeSet::new(); + for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + names.insert(name.to_string()); + } + } + Ok(names) +} + +/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. +fn records_a_frozen_edition(contents: &str) -> bool { + contents + .lines() + .any(|line| line.starts_with("min_vortex_version = ")) +} + +pub fn generate_editions() -> anyhow::Result<()> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../vortex/editions"); + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + + let session = EditionSession::empty(); + for declaration in EDITION_DECLARATIONS { + session + .declare(declaration) + .map_err(|error| anyhow!("declaring editions: {error}"))?; + } + session + .validate() + .map_err(|error| anyhow!("validating editions: {error}"))?; + + let mut expected = BTreeSet::new(); + for edition in session.editions() { + let path = dir.join(format!("{}.toml", edition.id)); + expected.insert(format!("{}.toml", edition.id)); + + // Freezing is permanent, and the record on disk is the only memory of it. Refusing + // here means unfreezing cannot be laundered through the exporter. + if edition.is_draft() + && let Ok(previous) = fs::read_to_string(&path) + && records_a_frozen_edition(&previous) + { + return Err(anyhow!( + "{} is recorded as frozen but its declaration is now a draft.\n\ + An edition that recorded a min_vortex_version carries a read-forever \ + guarantee and may never return to draft.", + edition.id, + )); + } + + fs::write(&path, record(&session, &edition)) + .with_context(|| format!("writing {}", path.display()))?; + } + + let strays: Vec = existing_records(&dir)? + .difference(&expected) + .cloned() + .collect(); + if !strays.is_empty() { + return Err(anyhow!( + "{} has records with no declared edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted; its declaration must stay in \ + `EDITION_DECLARATIONS`.", + dir.display(), + )); + } + + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1155ee3246a..772dd953e0e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod generate_editions; mod generate_fbs; mod generate_proto; use clap::Parser; +use crate::generate_editions::generate_editions; use crate::generate_fbs::generate_fbs; use crate::generate_proto::generate_proto; @@ -17,19 +19,23 @@ struct Xtask { #[derive(clap::Subcommand)] enum Commands { + /// Subcommand to regenerate the edition records under `vortex/editions`. + #[command(name = "generate-editions")] + Editions, /// Subcommand to regenerate flatbuffers language bindings for the Rust project. #[command(name = "generate-fbs")] - GenerateFlatbuffers, + Flatbuffers, /// Subcommand to regenerate protobuf language bindings for the Rust project. #[command(name = "generate-proto")] - GenerateProto, + Proto, } fn main() -> anyhow::Result<()> { let cli = Xtask::parse(); match cli.command { - Commands::GenerateFlatbuffers => generate_fbs()?, - Commands::GenerateProto => generate_proto()?, + Commands::Editions => generate_editions()?, + Commands::Flatbuffers => generate_fbs()?, + Commands::Proto => generate_proto()?, } Ok(()) } From fe888dfca586c5fc65b132acd5c3ce58ee613aee Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 13:59:09 +0000 Subject: [PATCH 5/8] Group the edition records by family Families version independently, so a flat directory mixed two unrelated chronologies and left the reader to spot the family from a filename prefix. Group the records as `vortex/editions//.toml`, mirroring the declarations in `vortex/src/editions`. The exporter creates a directory per family and treats a record under the wrong one as a stray, since that is what it is. The append-only check requires a newly added record's directory to match the family its name declares, so a record cannot be filed under a family whose chronology it does not extend. Signed-off-by: "Joe Isaacs" --- .github/scripts/check_edition_records.py | 19 +++++++-- vortex/editions/{ => core}/core2025.05.0.toml | 0 vortex/editions/{ => core}/core2025.06.0.toml | 0 vortex/editions/{ => core}/core2025.10.0.toml | 0 vortex/editions/{ => core}/core2026.07.0.toml | 0 vortex/editions/{ => core}/core2026.08.0.toml | 0 .../{ => unstable}/unstable2025.05.0.toml | 0 .../{ => unstable}/unstable2026.02.0.toml | 0 .../{ => unstable}/unstable2026.04.0.toml | 0 .../{ => unstable}/unstable2026.06.0.toml | 0 xtask/src/generate_editions.rs | 42 +++++++++++++------ 11 files changed, 45 insertions(+), 16 deletions(-) rename vortex/editions/{ => core}/core2025.05.0.toml (100%) rename vortex/editions/{ => core}/core2025.06.0.toml (100%) rename vortex/editions/{ => core}/core2025.10.0.toml (100%) rename vortex/editions/{ => core}/core2026.07.0.toml (100%) rename vortex/editions/{ => core}/core2026.08.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2025.05.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2026.02.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2026.04.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2026.06.0.toml (100%) diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index 9a3348e8b07..a15240b9c95 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -8,7 +8,8 @@ edit it in the same diff. A newly added record must also be newer than every edition already recorded for its family: -editions are only ever added going forward. +editions are only ever added going forward. Records are grouped by family, so +`vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. Usage: python3 check_edition_records.py --base origin/develop @@ -26,8 +27,8 @@ RECORD_DIR = "vortex/editions" -# `core2026.08.0.toml`: the file name is the edition id, so the record's identity is visible -# in the diff without reading the file. +# `core/core2026.08.0.toml`: the file name is the edition id and its directory is the family, +# so a record's identity is visible in the diff without reading the file. RECORD_NAME = re.compile( r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" ) @@ -79,7 +80,8 @@ def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: if match is None: sys.exit( f"{RECORD_DIR}/{name} is not a valid record name.\n" - "Records are named after the edition they record, e.g. `core2026.08.0.toml`." + "Records are named after the edition they record, e.g. " + "`core/core2026.08.0.toml`." ) return match["family"], (int(match["year"]), int(match["month"]), int(match["version"])) @@ -163,6 +165,15 @@ def check(base: str) -> list[str]: f"recorded ({family}{recorded}). Editions may only be added going forward." ) + # A record's family decides which chronology it extends, so the directory it sits + # in has to agree with the family its name declares. + directory = Path(path).parent.name + if directory != family: + errors.append( + f"adds {name} under {directory}/, but it records a {family} edition; " + "records are grouped by family" + ) + # The file name is the edition's identity, so it has to agree with the content. edition = parse_record(Path(path).read_text(), path).get("edition") if edition is None: diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core/core2025.05.0.toml similarity index 100% rename from vortex/editions/core2025.05.0.toml rename to vortex/editions/core/core2025.05.0.toml diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core/core2025.06.0.toml similarity index 100% rename from vortex/editions/core2025.06.0.toml rename to vortex/editions/core/core2025.06.0.toml diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core/core2025.10.0.toml similarity index 100% rename from vortex/editions/core2025.10.0.toml rename to vortex/editions/core/core2025.10.0.toml diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core/core2026.07.0.toml similarity index 100% rename from vortex/editions/core2026.07.0.toml rename to vortex/editions/core/core2026.07.0.toml diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core/core2026.08.0.toml similarity index 100% rename from vortex/editions/core2026.08.0.toml rename to vortex/editions/core/core2026.08.0.toml diff --git a/vortex/editions/unstable2025.05.0.toml b/vortex/editions/unstable/unstable2025.05.0.toml similarity index 100% rename from vortex/editions/unstable2025.05.0.toml rename to vortex/editions/unstable/unstable2025.05.0.toml diff --git a/vortex/editions/unstable2026.02.0.toml b/vortex/editions/unstable/unstable2026.02.0.toml similarity index 100% rename from vortex/editions/unstable2026.02.0.toml rename to vortex/editions/unstable/unstable2026.02.0.toml diff --git a/vortex/editions/unstable2026.04.0.toml b/vortex/editions/unstable/unstable2026.04.0.toml similarity index 100% rename from vortex/editions/unstable2026.04.0.toml rename to vortex/editions/unstable/unstable2026.04.0.toml diff --git a/vortex/editions/unstable2026.06.0.toml b/vortex/editions/unstable/unstable2026.06.0.toml similarity index 100% rename from vortex/editions/unstable2026.06.0.toml rename to vortex/editions/unstable/unstable2026.06.0.toml diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 961c4ba0a12..41e538f3619 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -5,6 +5,8 @@ //! //! Every declared edition gets one TOML file recording what it contains: the identifier, the //! minimum Vortex version whose reader supports it once frozen, and its full encoding set. +//! Records are grouped by family — `vortex/editions/core/core2025.05.0.toml` — mirroring the +//! declarations in `vortex/src/editions`, since families version independently. //! //! A record's mutability follows its edition. A draft is still being assembled, so its record //! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a @@ -88,20 +90,33 @@ fn record(session: &EditionSession, edition: &Edition) -> String { lines.join("\n") } -/// The `*.toml` file names present in the record directory. +/// The records present on disk, as `family/edition.toml` paths relative to the record +/// directory. A record filed under the wrong family reads as a stray, which is what it is. fn existing_records(dir: &Path) -> anyhow::Result> { - let mut names = BTreeSet::new(); - for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { - let path = entry?.path(); - if path - .extension() - .is_some_and(|extension| extension == "toml") - && let Some(name) = path.file_name().and_then(|name| name.to_str()) + let mut records = BTreeSet::new(); + if !dir.exists() { + return Ok(records); + } + for family in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let family = family?.path(); + if !family.is_dir() { + continue; + } + for entry in + fs::read_dir(&family).with_context(|| format!("reading {}", family.display()))? { - names.insert(name.to_string()); + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Ok(relative) = path.strip_prefix(dir) + && let Some(relative) = relative.to_str() + { + records.insert(relative.to_string()); + } } } - Ok(names) + Ok(records) } /// A record carries a `min_vortex_version` exactly when the edition it records is frozen. @@ -127,8 +142,11 @@ pub fn generate_editions() -> anyhow::Result<()> { let mut expected = BTreeSet::new(); for edition in session.editions() { - let path = dir.join(format!("{}.toml", edition.id)); - expected.insert(format!("{}.toml", edition.id)); + let relative = format!("{}/{}.toml", edition.id.family, edition.id); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(edition.id.family)) + .with_context(|| format!("creating the {} record directory", edition.id.family))?; // Freezing is permanent, and the record on disk is the only memory of it. Refusing // here means unfreezing cannot be laundered through the exporter. From cb141dbe9fbd3194b8b8a5daf9442be4d3091691 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 10:56:35 +0000 Subject: [PATCH 6/8] Move the edition declarations into vortex-edition, and check records with pygit2 The declarations name encodings by id string and import nothing but the types in vortex-edition, so nothing kept them in the vortex facade. Moving them makes the exporter cheap: xtask depended on vortex to reach EDITION_DECLARATIONS, which dragged 61 vortex crates and 338 packages into a build that also serves generate-fbs and generate-proto. It now pulls 4 and 81. `vortex::editions` re-exports every moved item, so its public API is unchanged, and it keeps the session wiring that does need the facade. The record check drove git through subprocess, parsing --name-status -z by hand. Read the object database with pygit2 instead: revisions resolve through revparse, renames come from the diff's own similarity detection rather than status-letter parsing, and record contents are read from the commit trees, so the check sees committed state only and never the working tree. Run it with `uv run --script`, which resolves the dependency from the script's inline metadata. Signed-off-by: "Joe Isaacs" --- .github/scripts/check_edition_records.py | 234 +++++++++++------- .github/workflows/ci.yml | 6 +- Cargo.lock | 2 +- .../src/declarations}/core/mod.rs | 0 .../src/declarations}/core/v2025_05.rs | 6 +- .../src/declarations}/core/v2025_06.rs | 6 +- .../src/declarations}/core/v2025_10.rs | 6 +- .../src/declarations}/core/v2026_07.rs | 6 +- .../src/declarations}/core/v2026_08.rs | 6 +- vortex-edition/src/declarations/mod.rs | 30 +++ .../src/declarations}/unstable/mod.rs | 0 .../src/declarations}/unstable/v2025_05.rs | 6 +- .../src/declarations}/unstable/v2026_02.rs | 6 +- .../src/declarations}/unstable/v2026_04.rs | 6 +- .../src/declarations}/unstable/v2026_06.rs | 6 +- vortex-edition/src/lib.rs | 2 + vortex/src/editions/mod.rs | 46 ++-- xtask/Cargo.toml | 2 +- xtask/src/generate_editions.rs | 8 +- 19 files changed, 226 insertions(+), 158 deletions(-) rename {vortex/src/editions => vortex-edition/src/declarations}/core/mod.rs (100%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2025_05.rs (92%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2025_06.rs (86%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2025_10.rs (87%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2026_07.rs (85%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2026_08.rs (85%) create mode 100644 vortex-edition/src/declarations/mod.rs rename {vortex/src/editions => vortex-edition/src/declarations}/unstable/mod.rs (100%) rename {vortex/src/editions => vortex-edition/src/declarations}/unstable/v2025_05.rs (85%) rename {vortex/src/editions => vortex-edition/src/declarations}/unstable/v2026_02.rs (85%) rename {vortex/src/editions => vortex-edition/src/declarations}/unstable/v2026_04.rs (88%) rename {vortex/src/editions => vortex-edition/src/declarations}/unstable/v2026_06.rs (85%) diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index a15240b9c95..ec4447c241e 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = ["pygit2>=1.14"] +# /// """Check that frozen edition records under `vortex/editions` never change. A record's mutability follows its edition. A draft is still being assembled, so its record may @@ -11,19 +15,24 @@ editions are only ever added going forward. Records are grouped by family, so `vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. +Both revisions are read straight out of the object database, so the check sees committed +state only and never the working tree. + Usage: - python3 check_edition_records.py --base origin/develop + uv run --script .github/scripts/check_edition_records.py --base origin/develop """ from __future__ import annotations import argparse import re -import subprocess import sys import tomllib -from pathlib import Path -from typing import Any +from pathlib import PurePosixPath +from typing import Any, Iterator + +import pygit2 +from pygit2.enums import DeltaStatus RECORD_DIR = "vortex/editions" @@ -36,42 +45,55 @@ # A record carries this exactly when the edition it records is frozen. FROZEN_MARKER = "min_vortex_version" +# Every way a record can change other than being added. Renames and copies carry an old path +# and a new one; the rest carry one. +CHANGE_VERBS = { + DeltaStatus.MODIFIED: "modifies", + DeltaStatus.DELETED: "deletes", + DeltaStatus.RENAMED: "renames", + DeltaStatus.COPIED: "copies", + DeltaStatus.TYPECHANGE: "retypes", +} + REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" - " vortex/src/editions// and regenerate the records with\n" + " vortex-edition/src/declarations// and regenerate the records with\n" " `cargo run -p xtask -- generate-editions`." ) -def git(*args: str) -> str: - result = subprocess.run(["git", *args], capture_output=True, text=True, check=False) - if result.returncode != 0: - sys.exit(f"git {' '.join(args)} failed:\n{result.stderr.strip()}") - return result.stdout +def parse_record(text: str, where: str) -> dict[str, Any]: + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + sys.exit(f"{where} is not valid TOML: {error}") -def merge_base(base: str) -> str: - result = subprocess.run( - ["git", "merge-base", base, "HEAD"], capture_output=True, text=True, check=False - ) - if result.returncode != 0: - sys.exit( - f"cannot find a merge base between {base} and HEAD:\n" - f"{result.stderr.strip()}\n" - "The checkout is probably too shallow; this check needs `fetch-depth: 0`." - ) - return result.stdout.strip() +def read_record(commit: pygit2.Commit, path: str) -> dict[str, Any] | None: + """Parse a record out of a commit's tree, or None when it holds no such file.""" + try: + blob = commit.tree[path] + except KeyError: + return None + return parse_record(blob.data.decode(), f"{path} at {commit.short_id}") -def parse_record(text: str, path: str) -> dict[str, Any]: - try: - return tomllib.loads(text) - except tomllib.TOMLDecodeError as error: - sys.exit(f"{path} is not valid TOML: {error}") +def record_paths(commit: pygit2.Commit) -> Iterator[str]: + """Every record path in a commit, relative to the repository root.""" + def walk(tree: pygit2.Tree, prefix: str) -> Iterator[str]: + for entry in tree: + path = f"{prefix}/{entry.name}" + if isinstance(entry, pygit2.Tree): + yield from walk(entry, path) + elif entry.name.endswith(".toml"): + yield path -def record_at(base: str, path: str) -> dict[str, Any]: - return parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") + try: + records = commit.tree[RECORD_DIR] + except KeyError: + return + yield from walk(records, RECORD_DIR) def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: @@ -86,36 +108,24 @@ def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: return match["family"], (int(match["year"]), int(match["month"]), int(match["version"])) -def changed_records(base: str) -> list[tuple[str, list[str]]]: - """The status and paths of every change to the record directory since `base`.""" - raw = git("diff", "--name-status", "-z", base, "HEAD", "--", RECORD_DIR) - fields = [field for field in raw.split("\0") if field] - changes: list[tuple[str, list[str]]] = [] - index = 0 - while index < len(fields): - status = fields[index] - # Renames and copies carry both the old and the new path. - count = 2 if status[0] in ("R", "C") else 1 - changes.append((status, fields[index + 1 : index + 1 + count])) - index += 1 + count - return changes +def changed_records(base: pygit2.Commit, head: pygit2.Commit) -> pygit2.Diff: + """The record directory's diff between two commits, with renames detected.""" + diff = base.tree.diff_to_tree(head.tree) + diff.find_similar() + return diff -def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: - """The newest edition already recorded for each family at `base`.""" +def newest_recorded(commit: pygit2.Commit) -> dict[str, tuple[int, int, int]]: + """The newest edition already recorded for each family at `commit`.""" newest: dict[str, tuple[int, int, int]] = {} - listing = git("ls-tree", "-r", "--name-only", base, "--", RECORD_DIR) - for path in listing.splitlines(): - family, key = parse_name(Path(path).name) + for path in record_paths(commit): + family, key = parse_name(PurePosixPath(path).name) newest[family] = max(key, newest.get(family, (0, 0, 0))) return newest -def check_modification(before: dict[str, Any], path: str) -> list[str]: +def check_modification(before: dict[str, Any], after: dict[str, Any], name: str) -> list[str]: """A frozen record may not change at all; name the fields that did.""" - name = Path(path).name - after = parse_record(Path(path).read_text(), path) - changed = sorted( key for key in before.keys() | after.keys() if before.get(key) != after.get(key) ) @@ -129,60 +139,80 @@ def check_modification(before: dict[str, Any], path: str) -> list[str]: return [f"modifies the frozen record {name}: {', '.join(changed)}"] -def check(base: str) -> list[str]: +def check_addition( + path: str, record: dict[str, Any], newest: dict[str, tuple[int, int, int]] +) -> list[str]: + """A new record must extend its family's chronology, and be filed under it.""" + errors = [] + name = PurePosixPath(path).name + family, key = parse_name(name) + + previous = newest.get(family) + if previous is not None and key <= previous: + recorded = f"{previous[0]}.{previous[1]:02}.{previous[2]}" + errors.append( + f"adds {name}, which is not newer than the {family} edition already " + f"recorded ({family}{recorded}). Editions may only be added going forward." + ) + + # A record's family decides which chronology it extends, so the directory it sits in has + # to agree with the family its name declares. + directory = PurePosixPath(path).parent.name + if directory != family: + errors.append( + f"adds {name} under {directory}/, but it records a {family} edition; " + "records are grouped by family" + ) + + # The file name is the edition's identity, so it has to agree with the content. + edition = record.get("edition") + if edition is None: + errors.append(f"adds {name}, which has no `edition` field") + elif edition != name.removesuffix(".toml"): + errors.append( + f"adds {name}, which records edition {edition!r}; the file name must be the edition id" + ) + return errors + + +def under_record_dir(*paths: str | None) -> bool: + return any(path is not None and path.startswith(f"{RECORD_DIR}/") for path in paths) + + +def check(base: pygit2.Commit, head: pygit2.Commit) -> list[str]: errors: list[str] = [] added: list[str] = [] - for status, paths in changed_records(base): - if status == "A": - added.append(paths[0]) + for patch in changed_records(base, head): + delta = patch.delta + old_path, new_path = delta.old_file.path, delta.new_file.path + if not under_record_dir(old_path, new_path): + continue + + if delta.status == DeltaStatus.ADDED: + added.append(new_path) continue # Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and # then edit it. A draft's record is free to change, move, or go away with the draft. - before = record_at(base, paths[0]) - if FROZEN_MARKER not in before: + before = read_record(base, old_path) + if before is None or FROZEN_MARKER not in before: continue - if status == "M": - errors.extend(check_modification(before, paths[0])) + if delta.status == DeltaStatus.MODIFIED: + after = read_record(head, new_path) or {} + errors.extend(check_modification(before, after, PurePosixPath(new_path).name)) else: - verb = {"D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} - errors.append( - f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}" - ) + verb = CHANGE_VERBS.get(delta.status, "changes") + moved = old_path if old_path == new_path else f"{old_path} -> {new_path}" + errors.append(f"{verb} the frozen record {moved}") - newest = recorded_at(base) + newest = newest_recorded(base) for path in sorted(added): - name = Path(path).name - family, key = parse_name(name) - - previous = newest.get(family) - if previous is not None and key <= previous: - recorded = f"{previous[0]}.{previous[1]:02}.{previous[2]}" - errors.append( - f"adds {name}, which is not newer than the {family} edition already " - f"recorded ({family}{recorded}). Editions may only be added going forward." - ) - - # A record's family decides which chronology it extends, so the directory it sits - # in has to agree with the family its name declares. - directory = Path(path).parent.name - if directory != family: - errors.append( - f"adds {name} under {directory}/, but it records a {family} edition; " - "records are grouped by family" - ) - - # The file name is the edition's identity, so it has to agree with the content. - edition = parse_record(Path(path).read_text(), path).get("edition") - if edition is None: - errors.append(f"adds {name}, which has no `edition` field") - elif edition != name.removesuffix(".toml"): - errors.append( - f"adds {name}, which records edition {edition!r}; the file name " - "must be the edition id" - ) + record = read_record(head, path) + if record is None: + continue + errors.extend(check_addition(path, record, newest)) return errors @@ -196,10 +226,24 @@ def main() -> int: ) args = parser.parse_args() - base = merge_base(args.base) - errors = check(base) + repo = pygit2.Repository(pygit2.discover_repository(".")) + try: + base_tip = repo.revparse_single(args.base).peel(pygit2.Commit) + except KeyError: + sys.exit(f"cannot resolve {args.base!r} in this repository") + + head = repo.head.peel(pygit2.Commit) + merge_base = repo.merge_base(base_tip.id, head.id) + if merge_base is None: + sys.exit( + f"{args.base} and HEAD have no common ancestor.\n" + "The checkout is probably too shallow; this check needs `fetch-depth: 0`." + ) + base = repo[merge_base] + + errors = check(base, head) if not errors: - print(f"{RECORD_DIR} preserves every frozen record against {args.base} ({base[:12]}).") + print(f"{RECORD_DIR} preserves every frozen record against {args.base} ({base.short_id}).") return 0 print(f"This change breaks the edition records in {RECORD_DIR}:\n", file=sys.stderr) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff41d03569f..ca95824c5fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,10 +73,14 @@ jobs: with: # The check compares against the merge base, so it needs real history. fetch-depth: 0 + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false - name: Check edition records run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" - python3 .github/scripts/check_edition_records.py --base "$BASE" + uv run --script .github/scripts/check_edition_records.py --base "$BASE" python-lint: name: "Python (lint)" diff --git a/Cargo.lock b/Cargo.lock index bdfab76e551..c51136dcbac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11269,7 +11269,7 @@ dependencies = [ "anyhow", "clap", "prost-build", - "vortex", + "vortex-edition", "xshell", ] diff --git a/vortex/src/editions/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs similarity index 100% rename from vortex/src/editions/core/mod.rs rename to vortex-edition/src/declarations/core/mod.rs diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex-edition/src/declarations/core/v2025_05.rs similarity index 92% rename from vortex/src/editions/core/v2025_05.rs rename to vortex-edition/src/declarations/core/v2025_05.rs index a25a6f971a4..115193619ba 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex-edition/src/declarations/core/v2025_05.rs @@ -3,9 +3,9 @@ //! The baseline `core` edition: stable encodings writable by Vortex 0.36.0. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The first edition of the `core` family, matching the first stable Vortex file release. pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex-edition/src/declarations/core/v2025_06.rs similarity index 86% rename from vortex/src/editions/core/v2025_06.rs rename to vortex-edition/src/declarations/core/v2025_06.rs index 015325b8429..7ebd3505799 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex-edition/src/declarations/core/v2025_06.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through June 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The June 2025 edition of the `core` family. pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex-edition/src/declarations/core/v2025_10.rs similarity index 87% rename from vortex/src/editions/core/v2025_10.rs rename to vortex-edition/src/declarations/core/v2025_10.rs index ae21026b595..6124c9e94b3 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex-edition/src/declarations/core/v2025_10.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through October 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The October 2025 edition of the `core` family. pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex-edition/src/declarations/core/v2026_07.rs similarity index 85% rename from vortex/src/editions/core/v2026_07.rs rename to vortex-edition/src/declarations/core/v2026_07.rs index 820c68cf7dd..78e0814d5a4 100644 --- a/vortex/src/editions/core/v2026_07.rs +++ b/vortex-edition/src/declarations/core/v2026_07.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through July 2026. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The July 2026 edition of the `core` family. pub const CORE_2026_07_0: EditionId = EditionId::new("core", 2026, 7, 0); diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex-edition/src/declarations/core/v2026_08.rs similarity index 85% rename from vortex/src/editions/core/v2026_08.rs rename to vortex-edition/src/declarations/core/v2026_08.rs index 4d47cfbe027..904dfb4ef73 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex-edition/src/declarations/core/v2026_08.rs @@ -3,9 +3,9 @@ //! The August 2026 core edition adding the canonical Map encoding. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The August 2026 core edition containing canonical Map arrays. pub const CORE_2026_08: EditionId = EditionId::new("core", 2026, 8, 0); diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs new file mode 100644 index 00000000000..0df0f58974c --- /dev/null +++ b/vortex-edition/src/declarations/mod.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The first-party Vortex edition declarations, one module per edition. +//! +//! These are plain constants naming encodings by id, so they depend on nothing but the types +//! in this crate. That keeps them cheap to read: tooling that only needs to know what an +//! edition contains — `cargo run -p xtask -- generate-editions`, for one — can depend on +//! this crate alone rather than on the whole of `vortex`. +//! +//! The `vortex` facade re-exports everything here and owns the session wiring: registering +//! the declarations and selecting which of them the default writer may emit. + +pub mod core; +pub mod unstable; + +use crate::EditionDeclaration; + +/// The first-party Vortex edition declarations. +pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ + &core::v2025_05::DECLARATION, + &core::v2025_06::DECLARATION, + &core::v2025_10::DECLARATION, + &core::v2026_07::DECLARATION, + &core::v2026_08::DECLARATION, + &unstable::v2025_05::DECLARATION, + &unstable::v2026_02::DECLARATION, + &unstable::v2026_04::DECLARATION, + &unstable::v2026_06::DECLARATION, +]; diff --git a/vortex/src/editions/unstable/mod.rs b/vortex-edition/src/declarations/unstable/mod.rs similarity index 100% rename from vortex/src/editions/unstable/mod.rs rename to vortex-edition/src/declarations/unstable/mod.rs diff --git a/vortex/src/editions/unstable/v2025_05.rs b/vortex-edition/src/declarations/unstable/v2025_05.rs similarity index 85% rename from vortex/src/editions/unstable/v2025_05.rs rename to vortex-edition/src/declarations/unstable/v2025_05.rs index 2cc6accdc38..1a992fd9937 100644 --- a/vortex/src/editions/unstable/v2025_05.rs +++ b/vortex-edition/src/declarations/unstable/v2025_05.rs @@ -3,9 +3,9 @@ //! The May 2025 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The May 2025 draft edition of the `unstable` family. pub const UNSTABLE_2025_05_0: EditionId = EditionId::new("unstable", 2025, 5, 0); diff --git a/vortex/src/editions/unstable/v2026_02.rs b/vortex-edition/src/declarations/unstable/v2026_02.rs similarity index 85% rename from vortex/src/editions/unstable/v2026_02.rs rename to vortex-edition/src/declarations/unstable/v2026_02.rs index e992cdee5ad..8e5faeeb28f 100644 --- a/vortex/src/editions/unstable/v2026_02.rs +++ b/vortex-edition/src/declarations/unstable/v2026_02.rs @@ -3,9 +3,9 @@ //! The February 2026 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The February 2026 draft edition of the `unstable` family. pub const UNSTABLE_2026_02_0: EditionId = EditionId::new("unstable", 2026, 2, 0); diff --git a/vortex/src/editions/unstable/v2026_04.rs b/vortex-edition/src/declarations/unstable/v2026_04.rs similarity index 88% rename from vortex/src/editions/unstable/v2026_04.rs rename to vortex-edition/src/declarations/unstable/v2026_04.rs index 90955f01d04..d64bc07529e 100644 --- a/vortex/src/editions/unstable/v2026_04.rs +++ b/vortex-edition/src/declarations/unstable/v2026_04.rs @@ -3,9 +3,9 @@ //! The April 2026 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The April 2026 draft edition of the `unstable` family. pub const UNSTABLE_2026_04_0: EditionId = EditionId::new("unstable", 2026, 4, 0); diff --git a/vortex/src/editions/unstable/v2026_06.rs b/vortex-edition/src/declarations/unstable/v2026_06.rs similarity index 85% rename from vortex/src/editions/unstable/v2026_06.rs rename to vortex-edition/src/declarations/unstable/v2026_06.rs index acbd739b656..560b3b5bf02 100644 --- a/vortex/src/editions/unstable/v2026_06.rs +++ b/vortex-edition/src/declarations/unstable/v2026_06.rs @@ -3,9 +3,9 @@ //! The June 2026 `unstable` encoding cohort. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The June 2026 draft edition of the `unstable` family. pub const UNSTABLE_2026_06_0: EditionId = EditionId::new("unstable", 2026, 6, 0); diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index a8eaf8e8287..0f71e520721 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -22,6 +22,7 @@ //! and enables them on the default session. See the published spec at //! . +pub mod declarations; mod session; pub mod test_harness; #[cfg(test)] @@ -33,6 +34,7 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +pub use declarations::EDITION_DECLARATIONS; pub use session::EditionSession; pub use session::EditionSessionExt; pub use session::EnabledEditions; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 88e4f351d0d..39a811c7203 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -3,21 +3,21 @@ //! The Vortex edition declarations. //! -//! [`vortex_edition`] provides the types, session variables, and test harness. The actual -//! first-party declarations live here, one module per edition. The default session first -//! registers them with [`crate::editions::register_default_editions`] and then selects its write -//! policy with [`crate::editions::enable_default_editions`]. +//! [`vortex_edition`] provides the types, session variables, test harness, and the +//! first-party declarations themselves. This module re-exports them and owns the session +//! wiring: the default session first registers them with +//! [`crate::editions::register_default_editions`] and then selects its write policy with +//! [`crate::editions::enable_default_editions`]. //! //! The default file writer resolves the session's enabled editions at write time. The //! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08`], and //! additionally enables the latest unstable edition when the `unstable_encodings` feature is //! selected. -pub mod core; #[cfg(test)] mod tests; -pub mod unstable; +pub use vortex_edition::EDITION_DECLARATIONS; pub use vortex_edition::Edition; pub use vortex_edition::EditionDeclaration; pub use vortex_edition::EditionId; @@ -25,20 +25,21 @@ pub use vortex_edition::EditionInclusion; pub use vortex_edition::EditionSession; pub use vortex_edition::EditionSessionExt; pub use vortex_edition::EnabledEditions; +pub use vortex_edition::declarations::core; +pub use vortex_edition::declarations::core::CORE_2025_05_0; +pub use vortex_edition::declarations::core::CORE_2025_06_0; +pub use vortex_edition::declarations::core::CORE_2025_10_0; +pub use vortex_edition::declarations::core::CORE_2026_07_0; +pub use vortex_edition::declarations::core::CORE_2026_08; +pub use vortex_edition::declarations::unstable; +pub use vortex_edition::declarations::unstable::UNSTABLE_2025_05_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_02_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_04_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_06_0; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_session::VortexSession; -pub use self::core::CORE_2025_05_0; -pub use self::core::CORE_2025_06_0; -pub use self::core::CORE_2025_10_0; -pub use self::core::CORE_2026_07_0; -pub use self::core::CORE_2026_08; -pub use self::unstable::UNSTABLE_2025_05_0; -pub use self::unstable::UNSTABLE_2026_02_0; -pub use self::unstable::UNSTABLE_2026_04_0; -pub use self::unstable::UNSTABLE_2026_06_0; - /// The `core` edition enabled for writing by the default Vortex session. pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; @@ -46,19 +47,6 @@ pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; /// `unstable_encodings` feature is selected. pub const DEFAULT_UNSTABLE_EDITION: EditionId = UNSTABLE_2026_06_0; -/// The first-party Vortex edition declarations. -pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ - &core::v2025_05::DECLARATION, - &core::v2025_06::DECLARATION, - &core::v2025_10::DECLARATION, - &core::v2026_07::DECLARATION, - &core::v2026_08::DECLARATION, - &unstable::v2025_05::DECLARATION, - &unstable::v2026_02::DECLARATION, - &unstable::v2026_04::DECLARATION, - &unstable::v2026_06::DECLARATION, -]; - /// Register the Vortex edition declarations with the session's [`EditionSession`]. pub fn register_default_editions(session: &VortexSession) { for declaration in EDITION_DECLARATIONS { diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 9878d2f6d1b..8bb8cd05125 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,7 +23,7 @@ test = false anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } prost-build = { workspace = true } -vortex = { workspace = true } +vortex-edition = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 41e538f3619..519545c887e 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -6,7 +6,7 @@ //! Every declared edition gets one TOML file recording what it contains: the identifier, the //! minimum Vortex version whose reader supports it once frozen, and its full encoding set. //! Records are grouped by family — `vortex/editions/core/core2025.05.0.toml` — mirroring the -//! declarations in `vortex/src/editions`, since families version independently. +//! declarations in `vortex-edition/src/declarations`, since families version independently. //! //! A record's mutability follows its edition. A draft is still being assembled, so its record //! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a @@ -22,9 +22,9 @@ use std::path::PathBuf; use anyhow::Context; use anyhow::anyhow; -use vortex::editions::EDITION_DECLARATIONS; -use vortex::editions::Edition; -use vortex::editions::EditionSession; +use vortex_edition::EDITION_DECLARATIONS; +use vortex_edition::Edition; +use vortex_edition::EditionSession; const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; From 0f6115c4d3ec36b4deecc9edc41f1086fe2d5381 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 10:56:58 +0000 Subject: [PATCH 7/8] Document the unstable edition family The spec described editions as frozen sets carrying a read-forever guarantee, then named unstable2026.06.0 once in passing without saying what the family is. It is the exception to the whole document: every unstable edition is a permanent draft, it never freezes, and it is only written when the unstable_encodings feature is selected. Signed-off-by: "Joe Isaacs" --- docs/specs/editions.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/specs/editions.md b/docs/specs/editions.md index 6dd87cd250d..77f13070d67 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -62,6 +62,19 @@ edition; each encoding's registry entry records the edition it joined in. In the encoding may be *deprecated*, meaning writers stop emitting it — but readers keep decoding it indefinitely, so deprecation never invalidates existing files. +## The `unstable` family + +Alongside `core` there is an `unstable` family, holding encodings that are still being +evaluated. It is the exception to everything above: every `unstable` edition is a permanent +draft, so the family never freezes and carries no read-compatibility guarantee at all. A file +written with these encodings is readable only by a build that knows them, and a future release +may stop supporting one. + +Because of that, the writer only emits them when you opt in — the default session enables the +newest `unstable` edition solely when the `unstable_encodings` cargo feature is selected. +Encodings graduate by being declared in a new `core` edition, which is where they pick up the +read-forever guarantee. + ## Edition registry Coming soon.. From be649b8b356bd0c0de96fd8eb8874365aae810a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 11:03:52 +0000 Subject: [PATCH 8/8] Declare edition families, and record what each one is for A family had no definition anywhere in the code: it was a bare string repeated on each EditionId, a directory name, and prose in two module doc comments and the spec, none of it reachable from an edition. Nothing validated the name either, so a typo minted a family of one whose editions were unordered against every real edition. Declare families explicitly. EditionFamily carries the name and what the family is for, EditionSession registers them beside editions, and validate() now rejects an edition whose family was never declared, or a family that documents nothing. The exporter writes a family.toml beside each family's editions. That record is documentation rather than a contract, so unlike an edition record it stays editable and the append-only check exempts it. Signed-off-by: "Joe Isaacs" --- .github/scripts/check_edition_records.py | 11 +++- vortex-edition/src/declarations/core/mod.rs | 11 ++++ vortex-edition/src/declarations/mod.rs | 4 ++ .../src/declarations/unstable/mod.rs | 12 ++++ vortex-edition/src/lib.rs | 36 ++++++++++++ vortex-edition/src/session.rs | 43 ++++++++++++++- vortex-edition/src/tests.rs | 46 ++++++++++++++++ vortex/editions/core/family.toml | 12 ++++ vortex/editions/unstable/family.toml | 13 +++++ vortex/src/editions/mod.rs | 12 +++- vortex/src/editions/tests.rs | 3 + xtask/src/generate_editions.rs | 55 +++++++++++++++++++ 12 files changed, 253 insertions(+), 5 deletions(-) create mode 100644 vortex/editions/core/family.toml create mode 100644 vortex/editions/unstable/family.toml diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index ec4447c241e..863e6c4b762 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -13,7 +13,8 @@ A newly added record must also be newer than every edition already recorded for its family: editions are only ever added going forward. Records are grouped by family, so -`vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. +`vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. The +`family.toml` beside them documents the family rather than pinning a contract, so it is exempt. Both revisions are read straight out of the object database, so the check sees committed state only and never the working tree. @@ -45,6 +46,10 @@ # A record carries this exactly when the edition it records is frozen. FROZEN_MARKER = "min_vortex_version" +# Beside each family's editions sits a record of the family itself. That one is documentation +# rather than a contract, so it stays editable and these rules leave it alone. +FAMILY_FILE = "family.toml" + # Every way a record can change other than being added. Renames and copies carry an old path # and a new one; the rest carry one. CHANGE_VERBS = { @@ -86,7 +91,7 @@ def walk(tree: pygit2.Tree, prefix: str) -> Iterator[str]: path = f"{prefix}/{entry.name}" if isinstance(entry, pygit2.Tree): yield from walk(entry, path) - elif entry.name.endswith(".toml"): + elif entry.name.endswith(".toml") and entry.name != FAMILY_FILE: yield path try: @@ -188,6 +193,8 @@ def check(base: pygit2.Commit, head: pygit2.Commit) -> list[str]: old_path, new_path = delta.old_file.path, delta.new_file.path if not under_record_dir(old_path, new_path): continue + if PurePosixPath(new_path).name == FAMILY_FILE: + continue if delta.status == DeltaStatus.ADDED: added.append(new_path) diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs index 4d05f11750c..c39585741f6 100644 --- a/vortex-edition/src/declarations/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -6,6 +6,17 @@ //! One module per edition, each declaring the edition and the encodings that join the //! family at it; members of earlier editions are inherited and never restated. +use crate::EditionFamily; + +/// The `core` family: what the default writer may emit. +pub static FAMILY: EditionFamily = EditionFamily { + name: "core", + doc: "The encodings the default file writer emits. Every core edition freezes, and a \ +frozen edition carries a read-forever guarantee: a file written with it stays readable by \ +every later Vortex release. New encodings join by being declared in a new edition; an \ +edition that has frozen never changes again.", +}; + pub mod v2025_05; pub mod v2025_06; pub mod v2025_10; diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs index 0df0f58974c..07dd540d0ba 100644 --- a/vortex-edition/src/declarations/mod.rs +++ b/vortex-edition/src/declarations/mod.rs @@ -15,6 +15,10 @@ pub mod core; pub mod unstable; use crate::EditionDeclaration; +use crate::EditionFamily; + +/// The first-party edition families. Every family must be declared before its editions. +pub static EDITION_FAMILIES: &[&EditionFamily] = &[&core::FAMILY, &unstable::FAMILY]; /// The first-party Vortex edition declarations. pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ diff --git a/vortex-edition/src/declarations/unstable/mod.rs b/vortex-edition/src/declarations/unstable/mod.rs index 5544ba45c0f..96ea60e4cd9 100644 --- a/vortex-edition/src/declarations/unstable/mod.rs +++ b/vortex-edition/src/declarations/unstable/mod.rs @@ -6,6 +6,18 @@ //! One module per draft edition, each declaring the encodings that join the family at it. //! Members of earlier editions are inherited and never restated. +use crate::EditionFamily; + +/// The `unstable` family: opt-in encodings with no compatibility guarantee. +pub static FAMILY: EditionFamily = EditionFamily { + name: "unstable", + doc: "Opt-in encodings that are still being evaluated. Every unstable edition stays a \ +draft, so the family never freezes and carries no compatibility guarantee: a file written \ +with these encodings is readable only by a build that knows them, and a later release may \ +stop supporting one. The writer emits them only when the `unstable_encodings` feature is \ +selected. An encoding graduates by joining a core edition.", +}; + pub mod v2025_05; pub mod v2026_02; pub mod v2026_04; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 0f71e520721..48e01ea7662 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -35,6 +35,7 @@ use std::fmt::Display; use std::fmt::Formatter; pub use declarations::EDITION_DECLARATIONS; +pub use declarations::EDITION_FAMILIES; pub use session::EditionSession; pub use session::EditionSessionExt; pub use session::EnabledEditions; @@ -111,6 +112,41 @@ impl Display for EditionId { } } +/// A family of editions: an independently versioned, additive group of encodings, registered +/// with [`EditionSession::declare_family`]. +/// +/// Every [`EditionId`] names one. Declaring the family is what makes the name real: +/// [`EditionSession::validate`] rejects an edition whose family was never declared, so a typo +/// cannot quietly mint a family of one. +#[derive(Clone, Copy, Debug)] +pub struct EditionFamily { + /// The family name, matching the [`EditionId::family`] of its editions, e.g. `core`. + pub name: &'static str, + /// What the family is for. Exported into the family's record, so a few sentences at + /// most: the long form belongs in the published spec. + pub doc: &'static str, +} + +impl EditionFamily { + /// Validate the family's form: a non-empty lowercase name and a non-empty doc. Checked + /// for every declared family by [`EditionSession::validate`]. + pub fn validate(&self) -> Result<(), EditionError> { + if self.name.is_empty() || !self.name.chars().all(|c| c.is_ascii_lowercase()) { + return Err(EditionError::new(format!( + "edition family {:?} must have a non-empty lowercase name, e.g. `core`", + self.name + ))); + } + if self.doc.trim().is_empty() { + return Err(EditionError::new(format!( + "edition family {} must document what it is for", + self.name + ))); + } + Ok(()) + } +} + /// An edition: a named set of encodings with a read-compatibility guarantee, registered with /// [`EditionSession::declare_edition`]. The set itself is computed from the registered /// [`EditionInclusion`]s by [`EditionSession::encodings_in`]. diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index e682590ba8f..2e1f5d0d9aa 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -17,6 +17,7 @@ use vortex_session::registry::Id; use crate::Edition; use crate::EditionDeclaration; use crate::EditionError; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::parse_release; @@ -35,6 +36,8 @@ pub struct EditionSession { #[derive(Debug, Default)] struct Inner { + /// Keyed by family name. + families: BTreeMap, /// Keyed by the display form of the edition id. editions: BTreeMap, /// Keyed by interned encoding id; ordered by the id's string form. @@ -90,6 +93,31 @@ impl EditionSession { Ok(()) } + /// Declare an edition family. Errors if a family with the same name is already + /// declared. Every family must be declared before [`EditionSession::validate`] will + /// accept editions belonging to it. + pub fn declare_family(&self, family: &EditionFamily) -> Result<(), EditionError> { + let mut inner = self.inner.write(); + if inner.families.contains_key(family.name) { + return Err(EditionError::new(format!( + "duplicate edition family {}", + family.name + ))); + } + inner.families.insert(family.name.to_string(), *family); + Ok(()) + } + + /// All declared families, sorted by name. + pub fn families(&self) -> Vec { + self.inner.read().families.values().copied().collect() + } + + /// Find a declared family by name. + pub fn find_family(&self, name: &str) -> Option { + self.inner.read().families.get(name).copied() + } + /// Declare an edition. Errors if an edition with the same id is already declared. pub fn declare_edition(&self, edition: Edition) -> Result<(), EditionError> { let mut inner = self.inner.write(); @@ -150,15 +178,26 @@ impl EditionSession { .collect() } - /// Validate all registered declarations. Errors on inclusions referencing undeclared - /// editions, editions out of chronological order within a family (unversioned drafts + /// Validate all registered declarations. Errors on editions in undeclared families, + /// inclusions referencing undeclared editions, editions out of chronological order within a family (unversioned drafts /// must be newest), malformed version strings, and members requiring a release newer /// than their edition declares. pub fn validate(&self) -> Result<(), EditionError> { let editions = self.editions(); + for family in self.families() { + family.validate()?; + } + for edition in &editions { edition.id.validate()?; + if self.find_family(edition.id.family).is_none() { + return Err(EditionError::new(format!( + "edition {} belongs to undeclared family {}; declare the family before \ + its editions", + edition.id, edition.id.family, + ))); + } if let Some(version) = edition.min_vortex_version && parse_release(version).is_none() { diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 09e1345615a..e023359760e 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -5,12 +5,23 @@ use vortex_session::VortexSession; use crate::Edition; use crate::EditionDeclaration; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::EditionSession; use crate::EditionSessionExt; use crate::EnabledEditions; +static TEST_FAMILY: EditionFamily = EditionFamily { + name: "test", + doc: "A family used by the unit tests.", +}; + +static OTHER_FAMILY: EditionFamily = EditionFamily { + name: "other", + doc: "A second family, for checking that families stay independent.", +}; + const FIRST: EditionId = EditionId::new("test", 2026, 1, 0); const SECOND: EditionId = EditionId::new("test", 2026, 7, 0); @@ -33,6 +44,9 @@ static DECLARATIONS: &[EditionDeclaration] = &[ fn session() -> EditionSession { let editions = EditionSession::empty(); + editions + .declare_family(&TEST_FAMILY) + .unwrap_or_else(|e| panic!("declaring the test family: {e}")); for declaration in DECLARATIONS { editions .declare(declaration) @@ -98,6 +112,7 @@ fn drafts_and_current() { // Freezing the first edition makes it current; the second stays a draft. let editions = EditionSession::empty(); + editions.declare_family(&TEST_FAMILY).unwrap(); editions .declare_edition(Edition { id: FIRST, @@ -179,8 +194,11 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi }; let session = VortexSession::empty().with::(); + session.editions().declare_family(&TEST_FAMILY)?; + session.editions().declare_family(&OTHER_FAMILY)?; session.register_edition(&DECLARATIONS[0])?; session.register_edition(&OTHER_DECLARATION)?; + session.editions().validate()?; session.enable_edition(FIRST)?; session.enable_edition(OTHER)?; @@ -274,3 +292,31 @@ fn edition_ids_order_within_family_only() { fn edition_id_display() { assert_eq!(FIRST.to_string(), "test2026.01.0"); } + +#[test] +fn families_must_be_declared_before_their_editions() -> Result<(), crate::EditionError> { + // An edition whose family was never declared: the name would otherwise be whatever the + // declaration happened to spell, and a typo would mint a family of one. + let editions = EditionSession::empty(); + editions.declare(&DECLARATIONS[0])?; + assert!(editions.validate().is_err()); + + editions.declare_family(&TEST_FAMILY)?; + editions.validate()?; + + // Declaring the same family twice is an error, as it is for editions. + assert!(editions.declare_family(&TEST_FAMILY).is_err()); + Ok(()) +} + +#[test] +fn families_must_document_themselves() { + let editions = EditionSession::empty(); + editions + .declare_family(&EditionFamily { + name: "undocumented", + doc: " ", + }) + .unwrap(); + assert!(editions.validate().is_err()); +} diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml new file mode 100644 index 00000000000..ebff1159dba --- /dev/null +++ b/vortex/editions/core/family.toml @@ -0,0 +1,12 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "core" + +doc = """ +The encodings the default file writer emits. Every core edition freezes, and a frozen +edition carries a read-forever guarantee: a file written with it stays readable by every +later Vortex release. New encodings join by being declared in a new edition; an edition that +has frozen never changes again. +""" diff --git a/vortex/editions/unstable/family.toml b/vortex/editions/unstable/family.toml new file mode 100644 index 00000000000..19c6b159176 --- /dev/null +++ b/vortex/editions/unstable/family.toml @@ -0,0 +1,13 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "unstable" + +doc = """ +Opt-in encodings that are still being evaluated. Every unstable edition stays a draft, so +the family never freezes and carries no compatibility guarantee: a file written with these +encodings is readable only by a build that knows them, and a later release may stop +supporting one. The writer emits them only when the `unstable_encodings` feature is +selected. An encoding graduates by joining a core edition. +""" diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 39a811c7203..2538a270901 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -18,8 +18,10 @@ mod tests; pub use vortex_edition::EDITION_DECLARATIONS; +pub use vortex_edition::EDITION_FAMILIES; pub use vortex_edition::Edition; pub use vortex_edition::EditionDeclaration; +pub use vortex_edition::EditionFamily; pub use vortex_edition::EditionId; pub use vortex_edition::EditionInclusion; pub use vortex_edition::EditionSession; @@ -47,8 +49,16 @@ pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; /// `unstable_encodings` feature is selected. pub const DEFAULT_UNSTABLE_EDITION: EditionId = UNSTABLE_2026_06_0; -/// Register the Vortex edition declarations with the session's [`EditionSession`]. +/// Register the Vortex edition families and declarations with the session's +/// [`EditionSession`]. pub fn register_default_editions(session: &VortexSession) { + for family in EDITION_FAMILIES { + session + .editions() + .declare_family(family) + .map_err(|e| vortex_err!("{e}")) + .vortex_expect("edition families are valid"); + } for declaration in EDITION_DECLARATIONS { session .register_edition(declaration) diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 46dcb73d53e..8a7bd100362 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -47,6 +47,9 @@ use super::UNSTABLE_2026_06_0; fn session() -> Result { let session = EditionSession::empty(); + for family in super::EDITION_FAMILIES { + session.declare_family(family)?; + } for declaration in EDITION_DECLARATIONS { session.declare(declaration)?; } diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 519545c887e..5a23739f092 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -23,7 +23,9 @@ use std::path::PathBuf; use anyhow::Context; use anyhow::anyhow; use vortex_edition::EDITION_DECLARATIONS; +use vortex_edition::EDITION_FAMILIES; use vortex_edition::Edition; +use vortex_edition::EditionFamily; use vortex_edition::EditionSession; const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; @@ -38,6 +40,44 @@ const DRAFT_NOTE: &str = "\ # record changes with it. Recording a min_vortex_version freezes the edition, after which # this file may never change again."; +/// The file recording what a family is, beside that family's editions. +const FAMILY_FILE: &str = "family.toml"; + +/// Render a family's record: its name and what it is for. Unlike an edition record this is +/// documentation, not a contract, so it stays editable. +fn family_record(family: &EditionFamily) -> String { + let mut lines = vec![ + GENERATED_BY.to_string(), + "# This describes the family the editions beside it belong to.".to_string(), + String::new(), + format!("name = \"{}\"", family.name), + String::new(), + "doc = \"\"\"".to_string(), + ]; + lines.extend(wrap(family.doc, 92)); + lines.extend(["\"\"\"".to_string(), String::new()]); + lines.join("\n") +} + +/// Wrap prose to a column, so a long doc reads as a paragraph rather than one endless line. +fn wrap(text: &str, width: usize) -> Vec { + let mut lines = Vec::new(); + let mut line = String::new(); + for word in text.split_whitespace() { + if !line.is_empty() && line.len() + 1 + word.len() > width { + lines.push(std::mem::take(&mut line)); + } + if !line.is_empty() { + line.push(' '); + } + line.push_str(word); + } + if !line.is_empty() { + lines.push(line); + } + lines +} + /// Render one edition's record. Deterministic: every list is sorted by encoding id, so the /// generated bytes depend only on the declarations. fn record(session: &EditionSession, edition: &Edition) -> String { @@ -131,6 +171,11 @@ pub fn generate_editions() -> anyhow::Result<()> { fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; let session = EditionSession::empty(); + for family in EDITION_FAMILIES { + session + .declare_family(family) + .map_err(|error| anyhow!("declaring edition families: {error}"))?; + } for declaration in EDITION_DECLARATIONS { session .declare(declaration) @@ -141,6 +186,16 @@ pub fn generate_editions() -> anyhow::Result<()> { .map_err(|error| anyhow!("validating editions: {error}"))?; let mut expected = BTreeSet::new(); + for family in session.families() { + let relative = format!("{}/{FAMILY_FILE}", family.name); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(family.name)) + .with_context(|| format!("creating the {} record directory", family.name))?; + fs::write(&path, family_record(&family)) + .with_context(|| format!("writing {}", path.display()))?; + } + for edition in session.editions() { let relative = format!("{}/{}.toml", edition.id.family, edition.id); let path = dir.join(&relative);