From 06add6bbde82c5ac5458d31dd063cc70b1ad6c13 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Wed, 9 Sep 2026 16:23:07 +0100 Subject: [PATCH] Editions that have not be stabilied can have members removed from them Signed-off-by: Robert Kruszewski --- docs/specs/versioning.md | 5 +- docs/specs/versioning/design.md | 33 +++-- docs/specs/versioning/editions.md | 65 ++++++--- docs/specs/versioning/using-editions.md | 60 ++++++++- .../src/decimal_byte_parts/array.rs | 2 +- .../src/decimal_byte_parts/plugin/tests.rs | 6 +- .../fastlanes/src/bitpacking/plugin/tests.rs | 2 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 2 +- encodings/fastlanes/src/for_/plugin/tests.rs | 2 +- encodings/fastlanes/src/for_/vtable/mod.rs | 2 +- encodings/parquet-variant/src/vtable.rs | 1 + encodings/zstd/src/editions.rs | 1 + vortex-array/src/array/vtable/mod.rs | 11 +- vortex-array/src/arrays/filter/vtable.rs | 2 +- vortex-array/src/arrays/interleave/mod.rs | 2 +- .../src/arrays/piecewise_sequence/tests.rs | 2 +- vortex-array/src/arrays/slice/vtable.rs | 2 +- vortex-array/src/session/mod.rs | 2 - vortex-cuda/src/layout.rs | 2 + vortex-edition/src/declarations/core/mod.rs | 4 +- .../src/declarations/core/v2025_05.rs | 1 + .../src/declarations/core/v2025_06.rs | 1 + .../src/declarations/core/v2025_10.rs | 1 + .../src/declarations/core/v2026_08.rs | 2 + .../src/declarations/core/v2026_08_2.rs | 1 + .../src/declarations/core/v2026_08_3.rs | 1 + vortex-edition/src/declarations/mod.rs | 5 +- .../src/declarations/preview/mod.rs | 11 +- .../src/declarations/preview/v2026_08.rs | 1 + .../src/declarations/unstable/mod.rs | 43 ++++++ vortex-edition/src/lib.rs | 54 +++++--- vortex-edition/src/session.rs | 72 +++++++--- vortex-edition/src/tests.rs | 53 ++++++++ vortex-file/src/lib.rs | 1 + vortex-file/src/tests.rs | 8 +- vortex-file/src/writer.rs | 4 + vortex-file/tests/common/mod.rs | 1 + vortex-json/src/editions.rs | 1 + vortex-layout/src/arrays/shared/vtable.rs | 2 +- .../zoned/aggregates/bloom_filter/mod.rs | 6 +- vortex-spatial/src/editions.rs | 1 + vortex-tensor/src/editions.rs | 1 + vortex/editions/core/family.toml | 8 +- vortex/editions/preview/family.toml | 7 +- vortex/editions/unstable/family.toml | 13 ++ vortex/editions/unstable/unstable.toml | 47 +++++++ vortex/src/editions/mod.rs | 5 + vortex/src/editions/tests.rs | 95 +++++++++++--- vortex/src/editions/unstable_tests.rs | 124 ++++++++++++++++++ xtask/Cargo.toml | 1 - xtask/src/check_editions.rs | 57 +++++++- xtask/src/generate_editions.rs | 63 +++++---- 52 files changed, 746 insertions(+), 153 deletions(-) create mode 100644 vortex-edition/src/declarations/unstable/mod.rs create mode 100644 vortex/editions/unstable/family.toml create mode 100644 vortex/editions/unstable/unstable.toml create mode 100644 vortex/src/editions/unstable_tests.rs diff --git a/docs/specs/versioning.md b/docs/specs/versioning.md index 9e8e9a1f5ab..4653588889b 100644 --- a/docs/specs/versioning.md +++ b/docs/specs/versioning.md @@ -75,7 +75,10 @@ forbids, or a custom strategy can choose an unsupported layout. The selection. [Draft editions](versioning/editions.md#draft-editions) have no frozen compatibility guarantee. -Custom wire formats written with edition checks disabled also fall outside the edition guarantee. +The [unstable edition](versioning/using-editions.md#unstable-edition), `UNSTABLE`, additionally +permits membership and wire contracts to change under existing IDs. It uses ordinary edition +selection and accepts plugin-defined members. Readers need matching implementations for those +formats. Custom wire formats written with edition checks disabled also fall outside the guarantee. ## Upgrades diff --git a/docs/specs/versioning/design.md b/docs/specs/versioning/design.md index 0efb20b6f32..3e21d3f4346 100644 --- a/docs/specs/versioning/design.md +++ b/docs/specs/versioning/design.md @@ -195,7 +195,8 @@ supports all its members. New formats require a later edition, leaving the earli to writers targeting older versions. An _edition family_ groups editions for related components. Membership is cumulative within a -family: each later edition includes all earlier members. +family: each later non-unstable edition includes all earlier non-unstable members. Unstable +editions have independent membership. The `core` family covers the default writer's formats, while optional features have independent families. A writer can select one `core` edition and one `tensor` edition, for example, and use the @@ -227,8 +228,8 @@ format, and reader version. formats, including those writers no longer choose. Edition selection does not restrict what a reader can read. 4. **Frozen edition records are immutable.** Membership, origin, and recorded minimum stay fixed. - Later editions include all earlier members within their family, while selecting editions from - multiple families permits the union of their members. + Later editions include earlier non-unstable members within their family, while selecting editions + from multiple families permits the union of their members. 5. **Edition enforcement covers the whole output.** Every serialized component must be permitted, including children and nested dependencies, so compressor declarations alone are insufficient. 6. **Recorded reader requirements are sound.** Each recorded origin version supplies readers for @@ -249,10 +250,19 @@ historical writing implementation. ## New formats -A new format starts in a draft edition so it can be tested before its origin commits to reading it -indefinitely. Drafts have no recorded minimum version or frozen guarantee. A format intended for -`core` can progress from its own family to `preview` for broader testing, then to `core` for default -use. Promotion preserves its wire ID and interpretation. The +A format whose contract is still changing belongs in `UNSTABLE`, an ordinary edition selection +without compatibility guarantees. Plugins register its members through ordinary edition inclusions. +Membership and wire contracts may change under existing IDs, so producers and readers need matching +implementations. The wire ID alone cannot distinguish earlier experimental contracts. + +Unstable membership is never inherited by another edition. Once a contract is settled, its owner +explicitly adds it to a new draft edition and removes its unstable inclusion. The ID can stay the +same, but this does not make earlier experimental files compatible. A format intended for `core` +enters `preview` for broader testing, then `core` for default use. Promotion preserves the settled +wire ID and contract. A contract frozen in any edition remains immutable regardless of whether +`UNSTABLE` also includes it; reader-visible corrections then require a new ID. + +Draft editions have no recorded minimum version or frozen guarantee. The [registry instructions](editions.md#format-testing-and-promotion) cover promotion, freezing, and recording the minimum version. @@ -261,10 +271,9 @@ recording the minimum version. The default BtrBlocks compressor excludes a scheme if any of its declared output wire IDs is forbidden. This filtering also applies to schemes used for child compression. -The current decimal scheme produces only single-child arrays and declares the v1 wire ID, so values -too wide for it remain in the standard uncompressed decimal representation. The multi-child -serializer exists, but the default scheme does not construct those arrays and no declared edition -permits their v2 ID. +By default, the decimal scheme uses the v1 wire format, leaving values too wide for a single child +in the standard uncompressed decimal representation. Selecting `UNSTABLE`, which includes the v2 +format, allows the default compressor to use its multi-child mode. ### Planned scheme configuration @@ -274,5 +283,5 @@ apply consistently to estimation, sampling, full compression, children, and fall per-writer scheme configuration is not implemented, and its API is unsettled. [^decimal-availability]: - No declared edition currently permits the multi-child format. The + The multi-child format requires selecting `UNSTABLE`. The [compression section](#compression) describes what the default compressor produces today. diff --git a/docs/specs/versioning/editions.md b/docs/specs/versioning/editions.md index 5666639e54d..21e6d86f33b 100644 --- a/docs/specs/versioning/editions.md +++ b/docs/specs/versioning/editions.md @@ -1,8 +1,8 @@ # Edition registry Each entry lists the components added by that edition, which also permits every component from -earlier editions in the same family. See [Using editions](using-editions.md) for configuration and -[Versioning design](design.md) for the compatibility rules. +earlier non-unstable editions in the same family. See [Using editions](using-editions.md) for +configuration and [Versioning design](design.md) for the compatibility rules. ## Frozen `core` editions @@ -113,22 +113,48 @@ For example, `core2026.08.0` declares `min`, `max`, `bounded_min`, `bounded_max` statistics, however, do store sums, in a fixed legacy field governed by the enclosing format's contract. +Aggregate serialization support alone does not permit an aggregate in persisted zone maps. Its ID +must belong to a selected edition, including `UNSTABLE` for experimental formats. The `all_*` +aggregates are in-memory pruning predicates and do not need entries. Legacy file statistics also +do not need entries. + +## Unstable edition + +`UNSTABLE` is a mutable edition with no minimum reader version or compatibility guarantee. Its +members and their wire contracts can change under the same IDs. It is selected using the ordinary +edition API and has no dated version. + +The built-in declaration contains: + +- `array`: `fastlanes.bitpacked.v2`, `fastlanes.delta`, `fastlanes.for.v2`, + `vortex.decimal_byte_parts.v2`, `vortex.patched`, `vortex.union` +- `layout`: `vortex.list` +- `aggregate`: `vortex.bloom_filter.sbbf`, `vortex.sum_v2` + +Plugins can register additional members through ordinary edition inclusions, with no central list +of allowed IDs. See [Unstable edition](using-editions.md#unstable-edition) for configuration. + ## Format testing and promotion -A format intended for `core` starts in a dedicated edition family. Its first edition is a draft, -with no recorded `min_library_version` and no frozen compatibility guarantee. Even at this draft -stage, the format is expected to be complete. If testing reveals a defect whose correction changes -what readers must understand, the correction needs a new wire ID and a later edition. +A format with a changing contract belongs in `UNSTABLE`. Its declaration can be added, changed, or +removed without a new edition or wire ID. Applications writing these formats need readers with +matching implementations. + +Once the contract is settled, remove its unstable inclusion and add it to a new draft edition. A +format intended for `core` enters `preview` for broader opt-in use, then a later `core` edition for +default use. Optional modules use their own families. The format can retain its ID, but earlier +experimental files acquire no compatibility guarantee. + +Unstable membership is never inherited. Ordinary edition membership remains additive, and a +contract frozen in any edition cannot change even if it also appears in `UNSTABLE`. A reader-visible +correction to a frozen format requires a new wire ID and a later edition. -After initial testing, the format can enter a new `preview` edition for broader opt-in use, then a -later `core` edition for default use. Promotion changes which editions permit the format while -preserving its contract and wire ID. The current `preview` edition is empty, so its first component -must go into a new edition. +The current `preview` edition is empty, so its first component must go into a new edition. ## Freezing an edition -A stable edition can freeze when its origin publishes the code that first supports all its members. -For `core`, this is a Vortex Rust crate release. Independent plugins use their own versions. +A non-unstable edition can freeze when its origin publishes the code that first supports all its +members. For `core`, this is a Vortex Rust crate release. Independent plugins use their own versions. Until the release version is known, the declaration uses `min_library_version: None`. Once it is known, the field records the first release that supports all members, usually while the next release @@ -144,10 +170,10 @@ Default declarations live in `vortex-edition/src/declarations/`. Optional module with their implementation code. The exported TOML records are under `vortex/editions/`, grouped by family. -1. For a new component, declare its own family and draft edition. For a revision, add a later - edition to the family that owns the earlier ID. -2. To promote a tested format, add it to new `preview` and `core` editions without changing its ID - or contract. +1. Register a changing format in `UNSTABLE`. Third-party plugins use ordinary edition inclusions. +2. Once its contract is settled, remove the unstable inclusion and add it to a new draft + edition. Formats intended for `core` pass through `preview`; optional modules use their own + families. Revisions to edition members require a new ID and a later edition. 3. When an edition freezes, record the first release of its origin that supports every permitted component, including inherited members. 4. Regenerate the records: @@ -156,6 +182,7 @@ family. cargo run -p xtask -- generate-editions ``` -CI's `check-editions` command rejects changes to frozen records, including renames, unfreezing, and -deletion, so changes to permitted wire formats require a later edition. The command also rejects a -new edition that does not follow its family's chronology. +CI's `check-editions` command rejects changes to frozen records, including renames, unfreezing, +relabeling as unstable, and deletion, so changes to permitted wire formats require a later edition. +The command also rejects a new ordinary edition that does not follow its family's chronology. +Unstable records are exempt from immutability and chronology checks. diff --git a/docs/specs/versioning/using-editions.md b/docs/specs/versioning/using-editions.md index db08812895e..a73bb99757b 100644 --- a/docs/specs/versioning/using-editions.md +++ b/docs/specs/versioning/using-editions.md @@ -50,7 +50,7 @@ can add formats without changing an application's `core` selection. A writer selects at most one edition per family. Selecting `core2026.08.0` and `tensor2026.04.0` permits every component in either edition. Within one family, a later edition includes all earlier -members. Across families, however, the selections are independent. +non-unstable members. Across families, however, the selections are independent. Check the [registry](editions.md#edition-registry) before enabling an optional family. For example, `tensor2026.04.0` is a draft and has no frozen minimum reader version. Adding it does not extend @@ -73,6 +73,62 @@ different origins require a separate version check for each project. In both cas register the required implementations, since meeting the version requirement alone does not make a plugin available. +## Unstable edition + +`UNSTABLE` is an ordinary edition selection with no compatibility guarantee. Its membership and +wire contracts may change under existing IDs, without creating a new edition. The default session +registers it but does not enable it. + +Enable it before constructing a compressor or starting a write: + +```rust +use vortex::VortexSessionDefault; +use vortex::editions::EditionSessionExt; +use vortex::editions::UNSTABLE; +use vortex::error::VortexResult; +use vortex::session::VortexSession; + +fn experimental_session() -> VortexResult { + let session = VortexSession::default(); + session.enable_edition(UNSTABLE)?; + Ok(session) +} +``` + +This permits the union of the selected `core` edition and `UNSTABLE`. The writer still checks every +serialized component, including children, layouts, extension dtypes, and stored aggregates. + +Plugins can add members using the ordinary edition registry; the built-in declarations are not a +closed list. For example, after registering the plugin's serializer and reader: + +```rust +use vortex::editions::ComponentKind; +use vortex::editions::EditionInclusion; +use vortex::editions::EditionSessionExt; +use vortex::editions::UNSTABLE; +use vortex::error::VortexResult; +use vortex::session::VortexSession; + +fn register_experiment(session: &VortexSession) -> VortexResult<()> { + session.editions().declare_inclusion(EditionInclusion::new( + ComponentKind::Array, + "my_plugin.experimental_encoding", + UNSTABLE, + )) +} +``` + +Registration does not enable the edition or install an implementation. Use +`session.editions().components_in(&UNSTABLE, kind)` to discover its registered members. + +Producers and readers of unstable formats need matching implementations. Selecting a frozen `core` +edition alongside `UNSTABLE` does not extend its guarantee to unstable formats. A contract already +frozen in any edition remains immutable, even if its ID is also registered in `UNSTABLE`. + +Unstable membership is independent and is never inherited by other editions. A settled format must +be explicitly added to an ordinary edition, and it can keep its ID. Earlier experimental files +remain outside the compatibility guarantee. + ## Write errors The writer rejects forbidden formats in arrays, children, layouts, nested extension dtypes, and @@ -102,6 +158,8 @@ array and a layout can share the same ID string while describing different forma required optional module. - For a draft edition, obtain a build that implements the component from the producer. A draft does not promise support in a published release. +- For an unstable format, obtain the matching implementation from the producer. The same ID in + another build may describe a different contract. - For a component absent from the registry, obtain its implementation from the producer and register it with the session. diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/array.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/array.rs index 93a228a3a87..6dc04e90326 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/array.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/array.rs @@ -259,7 +259,7 @@ impl VTable for DecimalByteParts { _array: ArrayView<'_, Self>, _session: &VortexSession, ) -> VortexResult>> { - vortex_bail!("DecimalByteParts serialization requires DecimalBytePartsPlugin") + Ok(None) } fn deserialize( diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/plugin/tests.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/plugin/tests.rs index e76476ac39e..39ded283a7d 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/plugin/tests.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/plugin/tests.rs @@ -250,11 +250,9 @@ fn bare_vtable_refuses_serde() -> VortexResult<()> { session.arrays().register(DecimalByteParts); let msp = msp(); let array = DecimalByteParts::try_new(msp.clone(), DecimalDType::new(19, 2))?.into_array(); - let result = session.array_serialize(&array); + let result = session.array_serialize(&array)?; assert!( - result.as_ref().is_err_and(|err| err - .to_string() - .contains("DecimalByteParts serialization requires DecimalBytePartsPlugin")), + result.is_none(), "expected unsupported VTable serialization, got {result:?}" ); diff --git a/encodings/fastlanes/src/bitpacking/plugin/tests.rs b/encodings/fastlanes/src/bitpacking/plugin/tests.rs index a5cf98400c3..2d81f5beda0 100644 --- a/encodings/fastlanes/src/bitpacking/plugin/tests.rs +++ b/encodings/fastlanes/src/bitpacking/plugin/tests.rs @@ -247,7 +247,7 @@ fn vtable_serde_requires_plugin() -> VortexResult<()> { let array = BitPackedData::encode(&values, 2, &mut SESSION.create_execution_ctx())?; let session = vortex_array::array_session(); session.arrays().register(BitPacked); - assert!(session.array_serialize(&array.into_array()).is_err()); + assert!(session.array_serialize(&array.into_array())?.is_none()); Ok(()) } diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index e92b1d040a2..e4ff6b061e9 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -170,7 +170,7 @@ impl VTable for BitPacked { _array: ArrayView<'_, Self>, _session: &VortexSession, ) -> VortexResult>> { - vortex_bail!("BitPacked serialization requires BitPackedPlugin") + Ok(None) } fn deserialize( diff --git a/encodings/fastlanes/src/for_/plugin/tests.rs b/encodings/fastlanes/src/for_/plugin/tests.rs index 6e490bd0568..03a888c3bc4 100644 --- a/encodings/fastlanes/src/for_/plugin/tests.rs +++ b/encodings/fastlanes/src/for_/plugin/tests.rs @@ -139,6 +139,6 @@ fn vtable_serde_requires_plugin() -> VortexResult<()> { )?; let session = vortex_array::array_session(); session.arrays().register(FoR); - assert!(session.array_serialize(&array.into_array()).is_err()); + assert!(session.array_serialize(&array.into_array())?.is_none()); Ok(()) } diff --git a/encodings/fastlanes/src/for_/vtable/mod.rs b/encodings/fastlanes/src/for_/vtable/mod.rs index ba2c353dec2..f9a318792d5 100644 --- a/encodings/fastlanes/src/for_/vtable/mod.rs +++ b/encodings/fastlanes/src/for_/vtable/mod.rs @@ -132,7 +132,7 @@ impl VTable for FoR { _array: ArrayView<'_, Self>, _session: &VortexSession, ) -> VortexResult>> { - vortex_bail!("FoR serialization requires FoRPlugin") + Ok(None) } fn deserialize( diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index f5933bfcc1b..f84c9a2775f 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -410,6 +410,7 @@ mod tests { editions .declare_edition(Edition { id: TEST_EDITION, + unstable: false, min_library_version: None, }) .map_err(|error| vortex_err!("{error}"))?; diff --git a/encodings/zstd/src/editions.rs b/encodings/zstd/src/editions.rs index 43f695a42e2..faeb0cd9f27 100644 --- a/encodings/zstd/src/editions.rs +++ b/encodings/zstd/src/editions.rs @@ -27,6 +27,7 @@ pub const ZSTD_2026_02: EditionId = EditionId::new("zstd", 2026, 2, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: ZSTD_2026_02, + unstable: false, min_library_version: None, }, added: &[EditionMember::array(&"vortex.zstd_buffers")], diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index e4a23ebe16c..edaabbea367 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -117,9 +117,14 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { /// Serialize encoding metadata into a byte buffer for IPC or file storage. /// - /// Return `None` if the array cannot be serialized by this encoding. Buffers and children are - /// serialized separately through [`buffer`](Self::buffer), [`nbuffers`](Self::nbuffers), and - /// child traversal. + /// Buffers and children are serialized separately through [`buffer`](Self::buffer), + /// [`nbuffers`](Self::nbuffers), and child traversal. + /// + /// Return `Ok(None)` if this encoding cannot be serialized. This includes in-memory-only + /// encodings and encodings whose wire format is owned by a separate + /// [`ArrayPlugin`](crate::array::ArrayPlugin). Do not return an error for this case: the + /// caller decides how to report an unserializable array. Return an error only when an array + /// that can be serialized fails to serialize. fn serialize( array: ArrayView<'_, Self>, session: &VortexSession, diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index b7642784fce..da5271b08bf 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -134,7 +134,7 @@ impl VTable for Filter { _session: &VortexSession, ) -> VortexResult>> { // TODO(joe): make this configurable - vortex_bail!("Filter array is not serializable") + Ok(None) } fn deserialize( diff --git a/vortex-array/src/arrays/interleave/mod.rs b/vortex-array/src/arrays/interleave/mod.rs index 28dfec7e5ee..49ed51f9ede 100644 --- a/vortex-array/src/arrays/interleave/mod.rs +++ b/vortex-array/src/arrays/interleave/mod.rs @@ -368,7 +368,7 @@ impl VTable for Interleave { _array: ArrayView<'_, Self>, _session: &VortexSession, ) -> VortexResult>> { - vortex_bail!("Interleave array is not serializable") + Ok(None) } fn deserialize( diff --git a/vortex-array/src/arrays/piecewise_sequence/tests.rs b/vortex-array/src/arrays/piecewise_sequence/tests.rs index a087404fee2..f1fb83f3257 100644 --- a/vortex-array/src/arrays/piecewise_sequence/tests.rs +++ b/vortex-array/src/arrays/piecewise_sequence/tests.rs @@ -171,7 +171,7 @@ fn serialization_is_not_supported() -> VortexResult<()> { .unwrap_err(); assert!( err.to_string() - .contains("Array vortex.piecewise-sequence does not support serialization"), + .contains("Array vortex.piecewise-sequence is not registered for serialization"), "{err}" ); Ok(()) diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index ef5cac4027c..74a1dcd2b8a 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -134,7 +134,7 @@ impl VTable for Slice { _session: &VortexSession, ) -> VortexResult>> { // TODO(joe): make this configurable - vortex_bail!("Slice array is not serializable") + Ok(None) } fn deserialize( diff --git a/vortex-array/src/session/mod.rs b/vortex-array/src/session/mod.rs index e9d0d36a55f..c1eb037de83 100644 --- a/vortex-array/src/session/mod.rs +++ b/vortex-array/src/session/mod.rs @@ -30,7 +30,6 @@ use crate::arrays::ListView; use crate::arrays::Map; use crate::arrays::Masked; use crate::arrays::Null; -use crate::arrays::PiecewiseSequence; use crate::arrays::Primitive; use crate::arrays::Struct; use crate::arrays::Union; @@ -105,7 +104,6 @@ impl Default for ArraySession { this.register(Dict); this.register(List); this.register(Masked); - this.register(PiecewiseSequence); this.register(VarBin); this diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index a8128963eee..6be0cfd36d0 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -597,6 +597,7 @@ const CUDA_EDITION: EditionId = EditionId::new("cuda", 2026, 9, 0); static CUDA_EDITION_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CUDA_EDITION, + unstable: false, min_library_version: None, }, added: &[EditionMember::layout(&"vortex.cuda_flat")], @@ -806,6 +807,7 @@ mod tests { session.register_edition(&EditionDeclaration { edition: Edition { id: OTHER_CUDA_EDITION, + unstable: false, min_library_version: None, }, added: &[], diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs index 93615b93dfc..23755a30048 100644 --- a/vortex-edition/src/declarations/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -16,8 +16,8 @@ pub static FAMILY: EditionFamily = EditionFamily { wire representation that old readers either recognize or reject; several IDs may deserialize \ into one current in-memory array. Every core edition freezes, and a frozen edition carries a \ read-forever guarantee: a file written with it stays readable by every later Vortex release. \ -New core objects first undergo testing in independently versioned families, then join preview for \ -broad opt-in use, and finally join core with the same IDs and wire contracts. An edition may \ +New core objects first undergo testing in UNSTABLE. Once their contracts \ +are settled, they join preview for broad opt-in use, then core with the same IDs and contracts. An edition may \ freeze in the release that cuts it; after that release version is known, the declaration is \ backfilled with it as the minimum. A frozen edition never changes.", }; diff --git a/vortex-edition/src/declarations/core/v2025_05.rs b/vortex-edition/src/declarations/core/v2025_05.rs index 38ca6d75a10..465cf0edc02 100644 --- a/vortex-edition/src/declarations/core/v2025_05.rs +++ b/vortex-edition/src/declarations/core/v2025_05.rs @@ -15,6 +15,7 @@ pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_05_0, + unstable: false, min_library_version: Some("0.36.0"), }, added: &[ diff --git a/vortex-edition/src/declarations/core/v2025_06.rs b/vortex-edition/src/declarations/core/v2025_06.rs index 3d033d280ed..c76a4f78f2f 100644 --- a/vortex-edition/src/declarations/core/v2025_06.rs +++ b/vortex-edition/src/declarations/core/v2025_06.rs @@ -15,6 +15,7 @@ pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_06_0, + unstable: false, min_library_version: Some("0.40.0"), }, added: &[ diff --git a/vortex-edition/src/declarations/core/v2025_10.rs b/vortex-edition/src/declarations/core/v2025_10.rs index cfe2a79f0c3..08850cf683e 100644 --- a/vortex-edition/src/declarations/core/v2025_10.rs +++ b/vortex-edition/src/declarations/core/v2025_10.rs @@ -15,6 +15,7 @@ pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_10_0, + unstable: false, min_library_version: Some("0.54.0"), }, added: &[ diff --git a/vortex-edition/src/declarations/core/v2026_08.rs b/vortex-edition/src/declarations/core/v2026_08.rs index a9022d8d569..8edf6e9f951 100644 --- a/vortex-edition/src/declarations/core/v2026_08.rs +++ b/vortex-edition/src/declarations/core/v2026_08.rs @@ -23,6 +23,7 @@ pub const CORE_2026_08_0: EditionId = EditionId::new("core", 2026, 8, 0); pub static DECLARATION_0: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_0, + unstable: false, min_library_version: Some("0.84.0"), }, added: &[ @@ -43,6 +44,7 @@ pub const CORE_2026_08_1: EditionId = EditionId::new("core", 2026, 8, 1); pub static DECLARATION_1: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_1, + unstable: false, min_library_version: Some("0.84.0"), }, added: &[EditionMember::array(&"vortex.onpair")], diff --git a/vortex-edition/src/declarations/core/v2026_08_2.rs b/vortex-edition/src/declarations/core/v2026_08_2.rs index 86222d23724..0a3430828fe 100644 --- a/vortex-edition/src/declarations/core/v2026_08_2.rs +++ b/vortex-edition/src/declarations/core/v2026_08_2.rs @@ -15,6 +15,7 @@ pub const CORE_2026_08_2: EditionId = EditionId::new("core", 2026, 8, 2); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_2, + unstable: false, min_library_version: Some("0.85.0"), }, added: &[EditionMember::array(&"vortex.map")], diff --git a/vortex-edition/src/declarations/core/v2026_08_3.rs b/vortex-edition/src/declarations/core/v2026_08_3.rs index f6f2bb1010b..f21d5fbd3ae 100644 --- a/vortex-edition/src/declarations/core/v2026_08_3.rs +++ b/vortex-edition/src/declarations/core/v2026_08_3.rs @@ -15,6 +15,7 @@ pub const CORE_2026_08_3: EditionId = EditionId::new("core", 2026, 8, 3); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_3, + unstable: false, min_library_version: Some("0.85.0"), }, added: &[ diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs index 39132db9283..efccb15bc6f 100644 --- a/vortex-edition/src/declarations/mod.rs +++ b/vortex-edition/src/declarations/mod.rs @@ -13,12 +13,14 @@ pub mod core; pub mod preview; +pub mod unstable; use crate::EditionDeclaration; use crate::EditionFamily; /// The first-party edition families. Every family must be declared before its editions. -pub static EDITION_FAMILIES: &[&EditionFamily] = &[&core::FAMILY, &preview::FAMILY]; +pub static EDITION_FAMILIES: &[&EditionFamily] = + &[&core::FAMILY, &preview::FAMILY, &unstable::FAMILY]; /// The first-party Vortex edition declarations. pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ @@ -30,4 +32,5 @@ pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ &core::v2026_08_2::DECLARATION, &core::v2026_08_3::DECLARATION, &preview::v2026_08::DECLARATION, + &unstable::DECLARATION, ]; diff --git a/vortex-edition/src/declarations/preview/mod.rs b/vortex-edition/src/declarations/preview/mod.rs index e203e35a475..16b2402b032 100644 --- a/vortex-edition/src/declarations/preview/mod.rs +++ b/vortex-edition/src/declarations/preview/mod.rs @@ -1,17 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The `preview` edition family: additive, opt-in components awaiting adoption into `core`. +//! The `preview` edition family: opt-in components awaiting adoption into `core`. use crate::EditionFamily; -/// The `preview` family: stable opt-in components not yet available by default. +/// The `preview` family: tested opt-in components not yet available by default. pub static FAMILY: EditionFamily = EditionFamily { name: "preview", origin: "vortex", - doc: "Additive, opt-in components maintained as part of Vortex but not yet adopted by the \ -default core writer. Components enter preview only once their serialized contracts are ready for \ -broad testing; independently evolving work remains in its own family until then.", + doc: "Opt-in components maintained as part of Vortex but not yet adopted by the default \ +core writer. Components leave UNSTABLE once their serialized contracts \ +are settled, then enter preview for broad testing. They join core with the same IDs and wire \ +contracts. Preview edition membership is additive; changing contracts remain in UNSTABLE.", }; pub mod v2026_08; diff --git a/vortex-edition/src/declarations/preview/v2026_08.rs b/vortex-edition/src/declarations/preview/v2026_08.rs index 94396ecf162..08bcb41d8c5 100644 --- a/vortex-edition/src/declarations/preview/v2026_08.rs +++ b/vortex-edition/src/declarations/preview/v2026_08.rs @@ -14,6 +14,7 @@ pub const PREVIEW_2026_08_0: EditionId = EditionId::new("preview", 2026, 8, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: PREVIEW_2026_08_0, + unstable: false, min_library_version: None, }, added: &[], diff --git a/vortex-edition/src/declarations/unstable/mod.rs b/vortex-edition/src/declarations/unstable/mod.rs new file mode 100644 index 00000000000..77bf11fbf88 --- /dev/null +++ b/vortex-edition/src/declarations/unstable/mod.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `unstable` edition family: experimental components with mutable wire contracts. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionFamily; +use crate::EditionId; +use crate::EditionMember; + +/// The shared family for experimental components, including third-party plugins. +pub static FAMILY: EditionFamily = EditionFamily { + name: "unstable", + origin: "vortex", + doc: "Opt-in experimental formats with no compatibility guarantee. Membership and wire \ +contracts may change under existing IDs. Plugins can register additional members using ordinary \ +edition inclusions. Settled formats move into preview or a plugin edition before freezing; \ +earlier experimental files acquire no compatibility guarantee.", +}; + +/// The shared mutable edition, with no compatibility guarantee. +pub const UNSTABLE: EditionId = EditionId::new("unstable", 0, 0, 0); + +/// Built-in serialized components awaiting stabilization in [`UNSTABLE`]. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE, + unstable: true, + min_library_version: None, + }, + added: &[ + EditionMember::array(&"fastlanes.bitpacked.v2"), + EditionMember::array(&"fastlanes.delta"), + EditionMember::array(&"fastlanes.for.v2"), + EditionMember::array(&"vortex.decimal_byte_parts.v2"), + EditionMember::array(&"vortex.patched"), + EditionMember::array(&"vortex.union"), + EditionMember::layout(&"vortex.list"), + EditionMember::aggregate(&"vortex.bloom_filter.sbbf"), + EditionMember::aggregate(&"vortex.sum_v2"), + ], +}; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index ede2646bb17..80b9302c4e1 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -2,24 +2,26 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors //! Definitions of Vortex *editions*: named sets of serialized component IDs. Frozen editions -//! carry a forever read-compatibility guarantee; draft editions do not. +//! carry a forever read-compatibility guarantee; draft and unstable editions do not. Unstable +//! editions permit membership and wire contracts to change under the same IDs. //! //! Editions live on the session, like encodings do: [`EditionSession`] holds the registered //! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations //! are plain constants — an [`EditionId`] plus an [`Edition`] record, and one //! [`EditionInclusion`] per member stating that it is a member of an edition *and every -//! later edition of the same family*. Any crate can register declarations into a session, -//! so inclusions can live next to the component they describe. +//! later non-unstable edition of the same family*. Unstable membership applies only to that edition. +//! Any crate can register declarations into a session, so inclusions can live next to the component +//! they describe. //! //! Every membership is typed by a [`ComponentKind`], and members are resolved one kind at a -//! time with [`EditionSessionExt::enabled_component_ids`]: the file writer restricts the -//! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets. Array -//! memberships name wire IDs rather than in-memory array representations. An array plugin may +//! time with [`EditionSessionExt::enabled_component_ids`], restricting arrays, layouts, extension +//! dtypes, and aggregates separately. Array memberships name wire IDs rather than in-memory array +//! representations. An array plugin may //! serialize one current in-memory representation under several historical IDs. The serialization //! context validates the ID chosen by the plugin before writing it. Readers resolve the ID stored //! in the file and either deserialize it into the current representation or reject it as unknown. //! -//! An edition is represented as a **draft** until its [`Edition::min_library_version`] is +//! A non-unstable edition is represented as a **draft** until its [`Edition::min_library_version`] is //! recorded. A stable edition may freeze in the release that cuts it; once that release version //! is known, the field is backfilled to document the freeze. The per-edition member sets are //! computed from the registered declarations by [`EditionSession::components_in`], and @@ -44,6 +46,7 @@ use std::fmt::Formatter; pub use declarations::EDITION_DECLARATIONS; pub use declarations::EDITION_FAMILIES; +pub use declarations::unstable::UNSTABLE; pub use session::EditionSession; pub use session::EditionSessionExt; pub use session::EnabledEditions; @@ -51,7 +54,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_session::registry::Id; -/// The identifier of an edition, e.g. `core2026.07.0`. +/// The identifier of an edition, e.g. `core2026.07.0`, or the undated [`UNSTABLE`]. /// /// The `family` names an independently versioned, additive group of members (`core` is the set /// available to the default writer). For `core`, the date components record when the edition @@ -70,8 +73,8 @@ pub struct EditionId { } impl EditionId { - /// Create an edition identifier. Validated by [`EditionId::validate`], which - /// [`test_harness::validate_edition`] exercises + /// Create a dated edition identifier. Use [`UNSTABLE`] for the shared unstable edition. + /// Validated by [`EditionId::validate`], which [`test_harness::validate_edition`] exercises /// per edition in unit tests. pub const fn new(family: &'static str, year: u16, month: u8, version: u8) -> Self { Self { @@ -90,10 +93,13 @@ impl EditionId { } /// Validate the identifier's form: a non-empty lowercase family, a four-digit year, - /// and a month in 01-12. Checked for every declared edition by - /// [`EditionSession::validate`] and per edition by + /// and a month in 01-12, except for the undated [`UNSTABLE`]. Checked for every declared edition + /// by [`EditionSession::validate`] and per edition by /// [`test_harness::validate_edition`]. pub fn validate(&self) -> VortexResult<()> { + if *self == UNSTABLE { + return Ok(()); + } if self.family.is_empty() || !self.family.chars().all(|c| c.is_ascii_lowercase()) { vortex_bail!("edition {self} must have a non-empty lowercase family, e.g. `core`"); } @@ -109,6 +115,9 @@ impl EditionId { impl Display for EditionId { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + if *self == UNSTABLE { + return f.write_str("unstable"); + } write!( f, "{}{}.{:02}.{}", @@ -197,8 +206,12 @@ impl Display for ComponentKind { pub struct Edition { /// The edition identifier. For a `core` edition, its date records when it freezes. pub id: EditionId, + /// Whether this edition permits changes to membership and wire contracts under existing IDs. + /// Unstable editions carry no compatibility guarantee and are not inherited by other editions. + /// This policy cannot replace the guarantee of an already frozen edition. + pub unstable: bool, /// The minimum version of the edition family's [`EditionFamily::origin`] whose reader - /// supports every member of this edition. + /// supports every member of this edition. Must be `None` for unstable editions. /// /// A stable edition may freeze in the release that cuts it. Until that release is cut, its /// version is not known and this remains `None`. The version is then backfilled to document @@ -211,17 +224,23 @@ pub struct Edition { } impl Edition { - /// A draft is an edition whose `min_library_version` has not been recorded yet. + /// Whether this edition carries a frozen read-compatibility guarantee. + pub fn is_frozen(&self) -> bool { + !self.unstable && self.min_library_version.is_some() + } + + /// A draft is a non-unstable edition whose `min_library_version` is not recorded yet. /// /// This describes the absence of a frozen compatibility guarantee, not necessarily the /// implementation stability of its members. pub fn is_draft(&self) -> bool { - self.min_library_version.is_none() + !self.unstable && self.min_library_version.is_none() } } /// Declares that a serialized component is a member of an edition — and of every later edition of -/// the same family. Registered with [`EditionSession::declare_inclusion`]. +/// the same family. Unstable membership applies only to that edition and is never inherited. +/// Registered with [`EditionSession::declare_inclusion`]. #[derive(Clone, Copy, Debug)] pub struct EditionInclusion { /// What the membership covers. Ids are unique per kind, so this is part of the @@ -324,7 +343,8 @@ pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, /// The members that join the family at this edition, each tagged with its [`ComponentKind`]. - /// Earlier entries are inherited and never restated. + /// Earlier non-unstable entries are inherited and never restated. Unstable editions list + /// their own independent membership. pub added: &'static [EditionMember], } diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index 4b192e3d7d9..12fd21d5a82 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -22,6 +22,7 @@ use crate::EditionDeclaration; use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; +use crate::UNSTABLE; use crate::parse_release; /// The session's registry of editions and edition inclusions. @@ -134,23 +135,29 @@ impl EditionSession { } /// Declare an edition inclusion. A component may belong to multiple families but joins each - /// family only once. A newer wire representation uses a new component ID. Kind is part of the + /// family only once, apart from independent unstable memberships. Kind is part of the /// key, so an array encoding and a layout may share an id. pub fn declare_inclusion(&self, inclusion: EditionInclusion) -> VortexResult<()> { let mut inner = self.inner.write(); - let by_id = inner.inclusions.entry(inclusion.kind).or_default(); - let history = by_id.entry(inclusion.component_id).or_default(); - let previous = history - .iter() - .filter(|existing| existing.since.family == inclusion.since.family) - .max_by_key(|existing| { - ( - existing.since.year, - existing.since.month, - existing.since.version, - ) + let unstable = inner + .editions + .get(&inclusion.since.to_string()) + .is_some_and(|e| e.unstable); + let previous = inner + .inclusions + .get(&inclusion.kind) + .and_then(|by_id| by_id.get(&inclusion.component_id)) + .into_iter() + .flatten() + .find(|existing| { + existing.since == inclusion.since + || (existing.since.family == inclusion.since.family + && !unstable + && !inner + .editions + .get(&existing.since.to_string()) + .is_some_and(|e| e.unstable)) }); - if let Some(previous) = previous { vortex_bail!( "{} {} already joined family {} in edition {}", @@ -160,6 +167,12 @@ impl EditionSession { previous.since, ); } + let history = inner + .inclusions + .entry(inclusion.kind) + .or_default() + .entry(inclusion.component_id) + .or_default(); history.push(inclusion); history.sort_by_key(|entry| { ( @@ -186,27 +199,40 @@ impl EditionSession { self.inner.read().editions.get(&id.to_string()).copied() } - /// The newest frozen edition of a family, if any. Drafts are never current. + /// The newest frozen edition of a family, if any. Draft and unstable editions are never current. pub fn current(&self, family: &str) -> Option { self.editions() .into_iter() - .rfind(|e| e.id.family == family && !e.is_draft()) + .rfind(|e| e.id.family == family && e.is_frozen()) } /// Compute an edition's members of one kind, sorted by component id. For each id, this returns /// its inclusion in the edition's family when it joined at or before the requested edition. + /// Unstable editions have independent membership and are never inherited. /// Only that kind's declarations are scanned. pub fn components_in(&self, edition: &EditionId, kind: ComponentKind) -> Vec { let inner = self.inner.read(); let Some(by_id) = inner.inclusions.get(&kind) else { return vec![]; }; + let unstable = inner + .editions + .get(&edition.to_string()) + .is_some_and(|e| e.unstable); by_id .values() .filter_map(|history| { history .iter() - .filter(|inclusion| inclusion.since.is_at_or_before(edition)) + .filter(|inclusion| { + inclusion.since == *edition + || (!unstable + && inclusion.since.is_at_or_before(edition) + && !inner + .editions + .get(&inclusion.since.to_string()) + .is_some_and(|e| e.unstable)) + }) .max_by_key(|inclusion| { ( inclusion.since.year, @@ -240,6 +266,15 @@ impl EditionSession { edition.id.family, ); } + if edition.unstable && edition.min_library_version.is_some() { + vortex_bail!( + "unstable edition {} cannot promise a minimum reader version", + edition.id + ); + } + if edition.id == UNSTABLE && !edition.unstable { + vortex_bail!("UNSTABLE must remain unstable; promote members into a new edition"); + } if let Some(version) = edition.min_library_version && parse_release(version).is_none() { @@ -252,9 +287,10 @@ impl EditionSession { // Within each family, frozen editions must precede drafts: a frozen edition after // an unversioned one would imply the draft was skipped. - for pair in editions.windows(2) { + let stable: Vec<_> = editions.iter().filter(|e| !e.unstable).collect(); + for pair in stable.windows(2) { let (prev, next) = (&pair[0], &pair[1]); - if prev.id.family == next.id.family && prev.is_draft() && !next.is_draft() { + if prev.id.family == next.id.family && prev.is_draft() && next.is_frozen() { vortex_bail!( "frozen edition {} follows draft {}; drafts must be newest in a family", next.id, diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 7394414499d..ec18d542cfe 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -35,6 +35,7 @@ static DECLARATIONS: &[EditionDeclaration] = &[ EditionDeclaration { edition: Edition { id: FIRST, + unstable: false, min_library_version: None, }, added: &[ @@ -45,6 +46,7 @@ static DECLARATIONS: &[EditionDeclaration] = &[ EditionDeclaration { edition: Edition { id: SECOND, + unstable: false, min_library_version: None, }, added: &[ @@ -148,12 +150,14 @@ fn drafts_and_current() { editions .declare_edition(Edition { id: FIRST, + unstable: false, min_library_version: Some("0.60.0"), }) .unwrap(); editions .declare_edition(Edition { id: SECOND, + unstable: false, min_library_version: None, }) .unwrap(); @@ -223,6 +227,7 @@ fn enabled_editions_are_independent_across_families() -> VortexResult<()> { static OTHER_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: OTHER, + unstable: false, min_library_version: None, }, added: &[EditionMember::array(&"other.delta")], @@ -250,6 +255,7 @@ fn serialized_array_ids_can_be_added_by_an_opt_in_family() -> VortexResult<()> { static OPT_IN_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: OPT_IN, + unstable: false, min_library_version: None, }, added: &[EditionMember::array(&"test.alpha_v2")], @@ -282,6 +288,7 @@ fn duplicate_declarations_error() { editions .declare_edition(Edition { id: FIRST, + unstable: false, min_library_version: None, }) .is_err() @@ -311,6 +318,7 @@ fn validate_rejects_inconsistent_declarations() -> VortexResult<()> { editions.declare_family(&TEST_FAMILY)?; editions.declare_edition(Edition { id: FIRST, + unstable: false, min_library_version: Some("0.70.0"), })?; editions.declare_inclusion(EditionInclusion { @@ -330,10 +338,12 @@ fn validate_rejects_inconsistent_declarations() -> VortexResult<()> { editions.declare_family(&TEST_FAMILY)?; editions.declare_edition(Edition { id: FIRST, + unstable: false, min_library_version: None, })?; editions.declare_edition(Edition { id: SECOND, + unstable: false, min_library_version: Some("0.70.0"), })?; assert!( @@ -352,6 +362,7 @@ fn validate_rejects_inconsistent_declarations() -> VortexResult<()> { editions.declare_family(&TEST_FAMILY)?; editions.declare_edition(Edition { id, + unstable: false, min_library_version: None, })?; let error = editions.validate().unwrap_err().to_string(); @@ -363,6 +374,7 @@ fn validate_rejects_inconsistent_declarations() -> VortexResult<()> { editions.declare_family(&TEST_FAMILY)?; editions.declare_edition(Edition { id: FIRST, + unstable: false, min_library_version: None, })?; editions.declare_inclusion(EditionInclusion::array("Test.ALPHA", FIRST))?; @@ -441,6 +453,7 @@ fn kinds_are_resolved_independently() -> VortexResult<()> { static MIXED: EditionDeclaration = EditionDeclaration { edition: Edition { id: FIRST, + unstable: false, min_library_version: None, }, added: &[ @@ -477,3 +490,43 @@ fn kinds_are_resolved_independently() -> VortexResult<()> { ); Ok(()) } + +/// Unstable membership is independent, so editing it cannot change a later frozen edition. +#[test] +fn unstable_members_require_explicit_promotion() -> VortexResult<()> { + const EXPERIMENT: EditionId = EditionId::new("test", 2026, 2, 0); + let editions = EditionSession::empty(); + editions.declare_family(&TEST_FAMILY)?; + for (id, unstable, min_library_version) in [ + (FIRST, false, Some("1.0.0")), + (EXPERIMENT, true, None), + (SECOND, false, Some("2.0.0")), + (EditionId::new("test", 2026, 8, 0), true, None), + ] { + editions.declare_edition(Edition { + id, + unstable, + min_library_version, + })?; + } + editions.declare_inclusion(EditionInclusion::array("test.original", FIRST))?; + editions.declare_inclusion(EditionInclusion::array("test.promoted", EXPERIMENT))?; + editions.declare_inclusion(EditionInclusion::array("test.changing", EXPERIMENT))?; + editions.declare_inclusion(EditionInclusion::array("test.promoted", SECOND))?; + editions.validate()?; + + let ids = |edition| { + editions + .components_in(&edition, ComponentKind::Array) + .into_iter() + .map(|inclusion| inclusion.component_id.to_string()) + .collect::>() + }; + assert_eq!(ids(EXPERIMENT), ["test.changing", "test.promoted"]); + assert_eq!(ids(SECOND), ["test.original", "test.promoted"]); + assert_eq!( + editions.current("test").map(|edition| edition.id), + Some(SECOND) + ); + Ok(()) +} diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 707de8bb47e..2372de06105 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -214,6 +214,7 @@ pub(crate) fn enable_all_registered_array_encodings(session: &VortexSession) { editions .declare_edition(Edition { id: TEST_EDITION, + unstable: false, min_library_version: None, }) .map_err(|error| vortex_err!("{error}")) diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index 3fddbbe4f77..ae45f772a13 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -87,6 +87,7 @@ use vortex_edition::EDITION_DECLARATIONS; use vortex_edition::EditionSession; use vortex_edition::EditionSessionExt; use vortex_edition::declarations::core::CORE_2026_08_3; +use vortex_edition::declarations::unstable::UNSTABLE; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_io::VortexWrite; @@ -289,7 +290,10 @@ async fn decimal_writer_uses_default_or_session_restrictions( } .build(); - for disable_editions in [false, true] { + for (disable_editions, unstable) in [(false, false), (true, false), (false, true)] { + if unstable { + session.enable_edition(UNSTABLE)?; + } let mut options = session.write_options(); if custom_strategy { options = options.with_strategy(Arc::clone(&strategy)); @@ -315,7 +319,7 @@ async fn decimal_writer_uses_default_or_session_restrictions( .as_opt::() .is_some_and(|parts| !parts.lower_parts().is_empty()) }); - assert_eq!(uses_v2, disable_editions && !custom_strategy); + assert_eq!(uses_v2, (disable_editions || unstable) && !custom_strategy); assert_arrays_eq!(array, actual, &mut session.create_execution_ctx()); } Ok(()) diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 2a6fb60847a..c4217ce3dde 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -784,6 +784,7 @@ mod tests { static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, + unstable: false, min_library_version: None, }, added: &[EditionMember::array(&"vortex.primitive")], @@ -807,6 +808,7 @@ mod tests { static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, + unstable: false, min_library_version: None, }, added: &[ @@ -846,6 +848,7 @@ mod tests { static ARRAYS_ONLY: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, + unstable: false, min_library_version: None, }, added: &[EditionMember::array(&"vortex.primitive")], @@ -890,6 +893,7 @@ mod tests { static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, + unstable: false, min_library_version: None, }, added: &[EditionMember::dtype(&"vortex.date")], diff --git a/vortex-file/tests/common/mod.rs b/vortex-file/tests/common/mod.rs index 29b8ac0024f..f3ab5fbe36a 100644 --- a/vortex-file/tests/common/mod.rs +++ b/vortex-file/tests/common/mod.rs @@ -21,6 +21,7 @@ pub fn enable_all_registered_array_encodings(session: &VortexSession) { editions .declare_edition(Edition { id: TEST_EDITION, + unstable: false, min_library_version: None, }) .map_err(|error| vortex_err!("{error}")) diff --git a/vortex-json/src/editions.rs b/vortex-json/src/editions.rs index 239d55042d6..f814f97ebe1 100644 --- a/vortex-json/src/editions.rs +++ b/vortex-json/src/editions.rs @@ -32,6 +32,7 @@ pub const JSON_2026_08: EditionId = EditionId::new("json", 2026, 8, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: JSON_2026_08, + unstable: false, min_library_version: None, }, added: &[EditionMember::dtype(&"vortex.json")], diff --git a/vortex-layout/src/arrays/shared/vtable.rs b/vortex-layout/src/arrays/shared/vtable.rs index e97505b0eb3..435de3b4271 100644 --- a/vortex-layout/src/arrays/shared/vtable.rs +++ b/vortex-layout/src/arrays/shared/vtable.rs @@ -104,7 +104,7 @@ impl VTable for Shared { _array: ArrayView<'_, Self>, _session: &VortexSession, ) -> VortexResult>> { - vortex_error::vortex_bail!("Shared array is not serializable") + Ok(None) } fn deserialize( diff --git a/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/mod.rs b/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/mod.rs index 3986f092ac6..979f4848d1c 100644 --- a/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/mod.rs +++ b/vortex-layout/src/layouts/zoned/aggregates/bloom_filter/mod.rs @@ -263,9 +263,9 @@ impl Display for BloomOptions { /// /// ### Stability /// -/// This aggregate is unstable. The hash function, the block layout, and the salt order can still -/// change without a file format version bump. Do not persist `vortex.bloom_filter.sbbf` partials -/// in a Vortex file. +/// This aggregate belongs to UNSTABLE, which must be explicitly enabled for writing. +/// Its contract may change under the same ID, including the hash function, block layout, and +/// salt order. Persisted filters require matching reader implementations. #[derive(Clone, Debug)] pub struct BloomFilter; diff --git a/vortex-spatial/src/editions.rs b/vortex-spatial/src/editions.rs index 298848a4be7..4fe54facd9b 100644 --- a/vortex-spatial/src/editions.rs +++ b/vortex-spatial/src/editions.rs @@ -32,6 +32,7 @@ pub const SPATIAL_2026_08: EditionId = EditionId::new("spatial", 2026, 8, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: SPATIAL_2026_08, + unstable: false, min_library_version: None, }, added: &[ diff --git a/vortex-tensor/src/editions.rs b/vortex-tensor/src/editions.rs index bd34ca7102a..ec5aad09ecc 100644 --- a/vortex-tensor/src/editions.rs +++ b/vortex-tensor/src/editions.rs @@ -28,6 +28,7 @@ pub const TENSOR_2026_04: EditionId = EditionId::new("tensor", 2026, 4, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: TENSOR_2026_04, + unstable: false, min_library_version: None, }, added: &[ diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml index 154d15bc994..379e03797de 100644 --- a/vortex/editions/core/family.toml +++ b/vortex/editions/core/family.toml @@ -10,8 +10,8 @@ The serialized components available to the default file writer. Each array ID na representation that old readers either recognize or reject; several IDs may deserialize into one current in-memory array. Every core edition freezes, and a frozen edition carries a read-forever guarantee: a file written with it stays readable by every later Vortex release. -New core objects first undergo testing in independently versioned families, then join -preview for broad opt-in use, and finally join core with the same IDs and wire contracts. An -edition may freeze in the release that cuts it; after that release version is known, the -declaration is backfilled with it as the minimum. A frozen edition never changes. +New core objects first undergo testing in UNSTABLE. Once their contracts are settled, they +join preview for broad opt-in use, then core with the same IDs and contracts. An edition may +freeze in the release that cuts it; after that release version is known, the declaration is +backfilled with it as the minimum. A frozen edition never changes. """ diff --git a/vortex/editions/preview/family.toml b/vortex/editions/preview/family.toml index 82c6a6a0239..75b9db22c2e 100644 --- a/vortex/editions/preview/family.toml +++ b/vortex/editions/preview/family.toml @@ -6,7 +6,8 @@ name = "preview" origin = "vortex" doc = """ -Additive, opt-in components maintained as part of Vortex but not yet adopted by the default -core writer. Components enter preview only once their serialized contracts are ready for -broad testing; independently evolving work remains in its own family until then. +Opt-in components maintained as part of Vortex but not yet adopted by the default core +writer. Components leave UNSTABLE once their serialized contracts are settled, then enter +preview for broad testing. They join core with the same IDs and wire contracts. Preview +edition membership is additive; changing contracts remain in UNSTABLE. """ diff --git a/vortex/editions/unstable/family.toml b/vortex/editions/unstable/family.toml new file mode 100644 index 00000000000..729df583498 --- /dev/null +++ b/vortex/editions/unstable/family.toml @@ -0,0 +1,13 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "unstable" +origin = "vortex" + +doc = """ +Opt-in experimental formats with no compatibility guarantee. Membership and wire contracts +may change under existing IDs. Plugins can register additional members using ordinary +edition inclusions. Settled formats move into preview or a plugin edition before freezing; +earlier experimental files acquire no compatibility guarantee. +""" diff --git a/vortex/editions/unstable/unstable.toml b/vortex/editions/unstable/unstable.toml new file mode 100644 index 00000000000..51f321d1763 --- /dev/null +++ b/vortex/editions/unstable/unstable.toml @@ -0,0 +1,47 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is unstable. Membership and wire contracts may change under existing IDs. +# This record carries no compatibility guarantee and may be changed or deleted. + +edition = "unstable" +family = "unstable" +origin = "vortex" +unstable = true + +# Components added by this edition. +[added] +arrays = [ + "fastlanes.bitpacked.v2", + "fastlanes.delta", + "fastlanes.for.v2", + "vortex.decimal_byte_parts.v2", + "vortex.patched", + "vortex.union", +] +layouts = [ + "vortex.list", +] +dtypes = [] +aggregates = [ + "vortex.bloom_filter.sbbf", + "vortex.sum_v2", +] + +# This edition's membership is independent and is never inherited. +[components] +arrays = [ + "fastlanes.bitpacked.v2", + "fastlanes.delta", + "fastlanes.for.v2", + "vortex.decimal_byte_parts.v2", + "vortex.patched", + "vortex.union", +] +layouts = [ + "vortex.list", +] +dtypes = [] +aggregates = [ + "vortex.bloom_filter.sbbf", + "vortex.sum_v2", +] diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 464cce2197b..65df6a10b66 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -43,6 +43,8 @@ pub use vortex_edition::declarations::core::CORE_2026_08_2; pub use vortex_edition::declarations::core::CORE_2026_08_3; pub use vortex_edition::declarations::preview; pub use vortex_edition::declarations::preview::PREVIEW_2026_08_0; +pub use vortex_edition::declarations::unstable; +pub use vortex_edition::declarations::unstable::UNSTABLE; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_session::VortexSession; @@ -81,3 +83,6 @@ pub fn enable_default_editions(session: &VortexSession) { .map_err(|e| vortex_err!("{e}")) .vortex_expect("default core edition is registered"); } + +#[cfg(all(test, feature = "files"))] +mod unstable_tests; diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index d65ad1e5e96..ce2b49a5842 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::collections::BTreeSet; + use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -9,6 +11,7 @@ use vortex_array::array_session; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::ListArray; +use vortex_array::arrays::Patched; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::assert_arrays_eq; @@ -39,6 +42,7 @@ use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; use vortex_io::session::RuntimeSession; use vortex_layout::session::LayoutSession; +use vortex_layout::session::LayoutSessionExt; use vortex_sequence::Sequence; use vortex_session::VortexSession; use vortex_session::registry::Id; @@ -52,6 +56,8 @@ use super::DEFAULT_CORE_EDITION; use super::DEFAULT_PREVIEW_EDITION; use super::EDITION_DECLARATIONS; use super::PREVIEW_2026_08_0; +use super::UNSTABLE; +use crate::VortexSessionDefault; fn session() -> VortexResult { let session = EditionSession::empty(); @@ -88,10 +94,10 @@ fn core_2026_08_1_dtype_set_is_pinned() { fn core_2026_08_2_is_frozen() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); assert!( - !session + session .find(&CORE_2026_08_2) .unwrap_or_else(|| panic!("{CORE_2026_08_2} is not registered")) - .is_draft() + .is_frozen() ); assert!( session @@ -111,10 +117,10 @@ fn core_2026_08_2_is_frozen() { fn core_2026_08_3_is_frozen_and_adds_variants() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); assert!( - !session + session .find(&CORE_2026_08_3) .unwrap_or_else(|| panic!("{CORE_2026_08_3} is not registered")) - .is_draft() + .is_frozen() ); assert!( session @@ -143,8 +149,8 @@ fn core_2026_08_3_is_frozen_and_adds_variants() { } #[test] -fn preview_starts_empty() { - let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); +fn preview_starts_empty() -> VortexResult<()> { + let session = session()?; for kind in [ ComponentKind::Array, ComponentKind::Layout, @@ -153,6 +159,41 @@ fn preview_starts_empty() { ] { assert!(session.components_in(&PREVIEW_2026_08_0, kind).is_empty()); } + Ok(()) +} + +#[test] +fn registered_array_and_layout_encodings_have_editions() { + let session = VortexSession::default(); + session.arrays().register(Patched); + let arrays = session + .arrays() + .registry() + .read(|registry| registry.keys().copied().collect::>()); + let layouts = session + .layouts() + .registry() + .read(|registry| registry.keys().copied().collect::>()); + for (kind, registered) in [ + (ComponentKind::Array, arrays), + (ComponentKind::Layout, layouts), + ] { + let editions = session.editions(); + let declared: BTreeSet = editions + .editions() + .iter() + .flat_map(|edition| editions.components_in(&edition.id, kind)) + .map(|inclusion| inclusion.component_id) + .collect(); + let missing: Vec = registered + .into_iter() + .filter(|id| !declared.contains(id)) + .collect(); + assert!( + missing.is_empty(), + "{kind} encodings without an edition: {missing:?}" + ); + } } #[test] @@ -212,8 +253,6 @@ fn core_2026_08_editions_add_onpair_before_map() { #[test] fn default_session_enables_the_write_editions() { - use crate::VortexSessionDefault; - let session = VortexSession::default(); let enabled = session.enabled_editions().editions(); assert!(enabled.contains(&DEFAULT_CORE_EDITION)); @@ -224,14 +263,38 @@ fn default_session_enables_the_write_editions() { ); assert!(!enabled.contains(&DEFAULT_PREVIEW_EDITION)); + assert!(!enabled.contains(&UNSTABLE)); +} + +#[test] +fn enabling_unstable_permits_only_persisted_aggregates() -> VortexResult<()> { + let session = VortexSession::default(); + let core_aggregates = session.enabled_component_ids(ComponentKind::Aggregate); + session.enable_edition(UNSTABLE)?; + let enabled = session.enabled_component_ids(ComponentKind::Aggregate); + assert!( + enabled + .iter() + .all(|id| !id.as_str().starts_with("vortex.all_")) + ); + let newly_enabled: Vec = enabled + .into_iter() + .filter(|id| !core_aggregates.contains(id)) + .collect(); + assert_eq!( + newly_enabled, + [ + Id::from("vortex.bloom_filter.sbbf"), + Id::from("vortex.sum_v2") + ] + ); + Ok(()) } #[test] fn core_edition_ids_are_registered_array_encodings() { use vortex_array::session::ArraySessionExt; - use crate::VortexSessionDefault; - let session = VortexSession::default(); let registry = session.arrays().registry().clone(); for inclusion in session @@ -250,8 +313,6 @@ fn core_edition_ids_are_registered_array_encodings() { fn core_dtype_ids_are_registered_extension_dtypes() { use vortex_array::dtype::session::DTypeSessionExt; - use crate::VortexSessionDefault; - let session = VortexSession::default(); let registry = session.dtypes().registry().clone(); for inclusion in session @@ -272,8 +333,6 @@ fn core_dtype_ids_are_registered_extension_dtypes() { fn core_aggregate_ids_are_registered_aggregate_fns() { use vortex_array::aggregate_fn::session::AggregateFnSessionExt; - use crate::VortexSessionDefault; - let session = VortexSession::default(); let declared = session .editions() @@ -300,8 +359,6 @@ fn core_aggregate_ids_are_registered_aggregate_fns() { /// maps record, for every dtype, or ordinary writes fail. #[tokio::test] async fn default_session_writes_every_default_zone_aggregate() -> VortexResult<()> { - use crate::VortexSessionDefault; - let session = VortexSession::default(); // Strings take the bounded min/max branch of the default aggregates, integers the plain // min/max branch, so one file exercises both. @@ -356,6 +413,7 @@ const WRITER_TEST_EDITION: EditionId = EditionId::new("writer-test", 2026, 7, 0) static WRITER_TEST_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: WRITER_TEST_EDITION, + unstable: false, min_library_version: None, }, added: &[ @@ -494,6 +552,7 @@ fn session_declaring(members: &[(ComponentKind, Id)]) -> VortexResult VortexResult<()> ); // Discover the layout tree through the fully declared core edition. - use crate::VortexSessionDefault; - let baseline = VortexSession::default(); let written = written_layout_ids(&baseline, write_with(&baseline, array.clone()).await?)?; assert!(written.len() > 1, "expected a layout tree, got {written:?}"); @@ -693,8 +750,6 @@ async fn serialization_context_rejects_unsupported_compressor_output() -> Vortex /// compressor itself from the edition. #[tokio::test] async fn serialization_context_accepts_supported_compressor_output() -> VortexResult<()> { - use crate::VortexSessionDefault; - let session = VortexSession::default(); let strategy = WriteStrategyBuilder::from_session(&session) .with_compressor(forbidden_sequence_compressor) diff --git a/vortex/src/editions/unstable_tests.rs b/vortex/src/editions/unstable_tests.rs new file mode 100644 index 00000000000..1bcbfd2eb71 --- /dev/null +++ b/vortex/src/editions/unstable_tests.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::ArrayDeserialization; +use vortex_array::ArrayId; +use vortex_array::ArrayPlugin; +use vortex_array::ArrayRef; +use vortex_array::ArraySerialization; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::Primitive; +use vortex_array::assert_arrays_eq; +use vortex_array::session::ArraySessionExt; +use vortex_array::stream::ArrayStreamExt; +use vortex_buffer::ByteBufferMut; +use vortex_buffer::buffer; +use vortex_edition::ComponentKind; +use vortex_edition::EditionInclusion; +use vortex_edition::EditionSessionExt; +use vortex_error::VortexResult; +use vortex_error::vortex_err; +use vortex_file::OpenOptionsSessionExt; +use vortex_file::WriteOptionsSessionExt; +use vortex_file::WriteStrategyBuilder; +use vortex_session::VortexSession; +use vortex_session::registry::Id; + +use super::UNSTABLE; +use crate::VortexSessionDefault; + +/// An external wire ID backed by the primitive implementation, registered only by this test. +#[derive(Debug)] +struct ExternalPlugin(Id); + +impl ArrayPlugin for ExternalPlugin { + fn id(&self) -> ArrayId { + ArrayPlugin::id(&Primitive) + } + + fn serialized_ids(&self) -> Vec { + vec![self.0] + } + + fn serialize( + &self, + array: &ArrayRef, + session: &VortexSession, + ) -> VortexResult> { + Ok( + ArrayPlugin::serialize(&Primitive, array, session)?.map(|mut serialized| { + serialized.serialized_id = self.0; + serialized + }), + ) + } + + fn deserialize( + &self, + parts: ArrayDeserialization<'_>, + session: &VortexSession, + ) -> VortexResult { + ArrayPlugin::deserialize( + &Primitive, + ArrayDeserialization { + serialized_id: ArrayPlugin::id(&Primitive), + ..parts + }, + session, + ) + } +} + +#[tokio::test] +async fn external_format_requires_opt_in_and_round_trips_with_its_original_id() -> VortexResult<()> +{ + let session = VortexSession::default(); + let id = Id::from("external.experimental_primitive"); + session.arrays().register(ExternalPlugin(id)); + session.editions().declare_inclusion(EditionInclusion::new( + ComponentKind::Array, + &id, + UNSTABLE, + ))?; + let array = buffer![1i32, 2, 3].into_array(); + let strategy = WriteStrategyBuilder::from_session(&session) + .with_compressor(|array: &ArrayRef, _: &mut ExecutionCtx| Ok(array.clone())) + .build(); + let mut rejected = ByteBufferMut::empty(); + let error = session + .write_options() + .with_strategy(strategy.clone()) + .write(&mut rejected, array.clone().to_array_stream()) + .await + .err() + .ok_or_else(|| vortex_err!("experimental output was accepted without an opt-in"))?; + assert!( + error + .to_string() + .contains("Serialized array ID external.experimental_primitive not permitted"), + "{error}" + ); + + session.enable_edition(UNSTABLE)?; + let mut bytes = ByteBufferMut::empty(); + session + .write_options() + .with_strategy(strategy) + .write(&mut bytes, array.clone().to_array_stream()) + .await?; + + // Reader permissions are independent: only the matching plugin is needed to read this file. + let reader = VortexSession::default(); + reader.arrays().register(ExternalPlugin(id)); + let actual = reader + .open_options() + .open_buffer(bytes)? + .scan()? + .into_array_stream()? + .read_all() + .await?; + assert_arrays_eq!(array, actual, &mut reader.create_execution_ctx()); + Ok(()) +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 843d026fb5d..00505dd868d 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -15,7 +15,6 @@ version = { workspace = true } [[bin]] name = "xtask" -test = false [dependencies] anyhow = { workspace = true } diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs index 2c57be104dd..f3f3b7331ce 100644 --- a/xtask/src/check_editions.rs +++ b/xtask/src/check_editions.rs @@ -3,12 +3,13 @@ //! Check that frozen edition records under `vortex/editions` never change. //! +//! An unstable record can change or be deleted without compatibility constraints. //! A draft record carries no read-forever guarantee and is not mechanically locked by this check. //! Once `min_library_version` is backfilled, the record captures the frozen edition's complete //! membership and may never change again. Whether a record was frozen is read from the base //! revision, so a change cannot unfreeze an edition and edit it in the same diff. //! -//! A newly added record must also be newer than every edition already recorded for its +//! A newly added non-unstable record must also be newer than every edition already recorded for its //! family: editions are only ever added going forward. Records are grouped by family, so //! `vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. The //! `family.toml` beside them documents the family rather than pinning a contract, so it is @@ -40,6 +41,14 @@ pub const RECORD_DIR: &str = "vortex/editions"; /// A record carries this key once the edition's freeze has been documented. const FROZEN_MARKER: &str = "min_library_version"; +pub(super) fn record_is_unstable(record: &Table) -> bool { + record.get("unstable").and_then(|value| value.as_bool()) == Some(true) +} + +pub(super) fn record_is_frozen(record: &Table) -> bool { + !record_is_unstable(record) && record.contains_key(FROZEN_MARKER) +} + const REMEDY: &str = "\ A frozen edition is immutable. To add component IDs, declare a NEW edition in the family's declaration module and regenerate the records with @@ -50,6 +59,9 @@ type Chronology = (u16, u8, u8); /// Split a record file name into its family and its place in that family's chronology. fn parse_name(name: &str) -> anyhow::Result<(&str, Chronology)> { + if name == "unstable.toml" { + return Ok(("unstable", (0, 0, 0))); + } let malformed = || { anyhow!( "{RECORD_DIR}/{name} is not a valid record name. Records are named after the \ @@ -113,6 +125,15 @@ fn newest_recorded( if !name.ends_with(".toml") || name == FAMILY_FILE { return TreeWalkResult::Ok; } + let record = repo.find_blob(entry.id()).ok().and_then(|blob| { + std::str::from_utf8(blob.content()) + .ok()? + .parse::() + .ok() + }); + if record.as_ref().is_some_and(record_is_unstable) { + return TreeWalkResult::Ok; + } match parse_name(name) { Ok((family, chronology)) => { let slot = newest.entry(family.to_string()).or_insert(chronology); @@ -170,7 +191,8 @@ fn check_addition( .ok_or_else(|| anyhow!("{path} has no file name"))?; let (family, chronology) = parse_name(name)?; - if let Some(previous) = newest.get(family) + if !record_is_unstable(record) + && let Some(previous) = newest.get(family) && chronology <= *previous { errors.push(format!( @@ -270,7 +292,7 @@ pub fn check_editions(base: &str) -> anyhow::Result<()> { let Some(before) = read_record(&repo, &base_commit, &old)? else { continue; }; - if !before.contains_key(FROZEN_MARKER) { + if !record_is_frozen(&before) { continue; } @@ -318,3 +340,32 @@ pub fn check_editions(base: &str) -> anyhow::Result<()> { .join("\n"); bail!("This change breaks the edition records in {RECORD_DIR}:\n\n{listed}\n\n{REMEDY}"); } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use toml::Table; + + use super::check_addition; + use super::check_modification; + use super::record_is_frozen; + + #[test] + fn unstable_records_do_not_weaken_frozen_contracts() -> anyhow::Result<()> { + let frozen: Table = "edition = 'core2026.08.0'\nmin_library_version = '0.84.0'".parse()?; + let mut changed = frozen.clone(); + changed.insert("unstable".into(), true.into()); + assert!(record_is_frozen(&frozen)); + assert!(!check_modification(&frozen, &changed, "core2026.08.0.toml").is_empty()); + + let unstable: Table = "edition = 'unstable'\norigin = 'vortex'\nunstable = true".parse()?; + assert!(!record_is_frozen(&unstable)); + let newest = BTreeMap::from([("unstable".into(), (2026, 8, 0))]); + assert!( + check_addition("vortex/editions/unstable/unstable.toml", &unstable, &newest)? + .is_empty() + ); + Ok(()) + } +} diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 42017461d57..921d04116ca 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -12,7 +12,7 @@ //! A draft record carries no compatibility guarantee. Once `min_library_version` is backfilled to //! document its freeze, the record carries a read-forever guarantee and may never change again. //! CI enforces that against git history with `cargo run -p xtask -- check-editions`; this exporter -//! enforces the two rules that history cannot see, refusing to delete a record or to unfreeze one. +//! also refuses to unfreeze a record or delete a non-unstable record. use std::collections::BTreeSet; use std::fs; @@ -32,6 +32,8 @@ use vortex_edition::EditionSession; use crate::check_editions::FAMILY_FILE; use crate::check_editions::RECORD_DIR; +use crate::check_editions::record_is_frozen; +use crate::check_editions::record_is_unstable; const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; @@ -40,6 +42,10 @@ const FROZEN_NOTE: &str = "\ # contains never changes again. Freezing a new edition adds a new file to this directory; # editing or deleting a frozen one is rejected by CI."; +const UNSTABLE_NOTE: &str = "\ +# This edition is unstable. Membership and wire contracts may change under existing IDs. +# This record carries no compatibility guarantee and may be changed or deleted."; + const DRAFT_NOTE: &str = "\ # This edition record has no read-forever guarantee. It may describe an evolving feature, # a stable component awaiting adoption, or a release waiting to be cut. New capabilities advance @@ -131,7 +137,9 @@ fn kind_lists( /// Render one edition's record. Deterministic: every list is sorted by component id, so the /// generated bytes depend only on the declarations. fn record(session: &EditionSession, edition: &Edition, family: &EditionFamily) -> String { - let note = if edition.is_draft() { + let note = if edition.unstable { + UNSTABLE_NOTE + } else if edition.is_draft() { DRAFT_NOTE } else { FROZEN_NOTE @@ -144,6 +152,9 @@ fn record(session: &EditionSession, edition: &Edition, family: &EditionFamily) - format!("family = \"{}\"", edition.id.family), format!("origin = \"{}\"", family.origin), ]; + if edition.unstable { + lines.push("unstable = true".to_string()); + } if let Some(min_library_version) = edition.min_library_version { lines.push(format!("min_library_version = \"{min_library_version}\"")); } @@ -159,13 +170,18 @@ fn record(session: &EditionSession, edition: &Edition, family: &EditionFamily) - .filter(|inclusion| inclusion.since == edition.id) .collect() }); - lines.extend([ - String::new(), - "# The edition's full membership: the members above, plus every member of earlier" - .to_string(), - "# editions of the family.".to_string(), - "[components]".to_string(), - ]); + lines.push(String::new()); + if edition.unstable { + lines + .push("# This edition's membership is independent and is never inherited.".to_string()); + } else { + lines.push( + "# The edition's full membership: the members above, plus every member of earlier" + .to_string(), + ); + lines.push("# editions of the family.".to_string()); + } + lines.push("[components]".to_string()); kind_lists(&mut lines, |kind| session.components_in(&edition.id, kind)); lines.push(String::new()); lines.join("\n") @@ -200,13 +216,6 @@ fn existing_records(dir: &Path) -> anyhow::Result> { Ok(records) } -/// A record carries a `min_library_version` once its freeze has been documented. -fn records_a_frozen_edition(contents: &str) -> bool { - contents - .lines() - .any(|line| line.starts_with("min_library_version = ")) -} - pub fn generate_editions() -> anyhow::Result<()> { let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("..") @@ -256,14 +265,14 @@ pub fn generate_editions() -> anyhow::Result<()> { // Freezing is permanent, and the record on disk is the only memory of it. Refusing // here means unfreezing cannot be laundered through the exporter. - if edition.is_draft() + if !edition.is_frozen() && let Ok(previous) = fs::read_to_string(&path) - && records_a_frozen_edition(&previous) + && record_is_frozen(&previous.parse()?) { return Err(anyhow!( - "{} is recorded as frozen but its declaration is now a draft.\n\ + "{} is recorded as frozen but its declaration no longer carries that guarantee.\n\ An edition that recorded a min_library_version carries a read-forever \ - guarantee and may never return to draft.", + guarantee and may never become draft or unstable.", edition.id, )); } @@ -283,9 +292,19 @@ pub fn generate_editions() -> anyhow::Result<()> { .difference(&expected) .cloned() .collect(); - if !strays.is_empty() { + let mut locked = Vec::new(); + for stray in strays { + let path = dir.join(&stray); + let previous = fs::read_to_string(&path)?.parse()?; + if record_is_unstable(&previous) { + fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?; + } else { + locked.push(stray); + } + } + if !locked.is_empty() { return Err(anyhow!( - "{} has records with no declared edition behind them: {strays:?}.\n\ + "{} has records with no declared edition behind them: {locked:?}.\n\ A frozen edition may never be deleted; its declaration must stay in the \ generator's edition declarations.", dir.display(),