Teach ArrayVTable about capabilities and let arrays provide extensible capabilities

encodings/fsst/src/array.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::any::TypeId;
 use std::fmt::Debug;
 use std::fmt::Display;
 use std::fmt::Formatter;
@@ -46,7 +47,9 @@ use vortex_array::serde::ArrayChildren;
 use vortex_array::validity::Validity;
 use vortex_array::vtable::VTable;
 use vortex_array::vtable::ValidityVTable;
+use vortex_array::vtable::VarBinExportable;
 use vortex_array::vtable::child_to_validity;
+use vortex_array::vtable::has_capability;
 use vortex_array::vtable::validity_to_child;
 use vortex_buffer::Buffer;
 use vortex_buffer::BufferMut;
@@ -114,6 +117,8 @@ impl ArrayEq for FSSTData {
     }
 }
 
+impl VarBinExportable for FSST {}
+
 impl VTable for FSST {
     type TypedArrayData = FSSTData;
     type OperationsVTable = Self;
@@ -318,6 +323,10 @@ impl VTable for FSST {
         canonicalize_fsst(array.as_view(), ctx).map(ExecutionResult::done)
     }
 
+    fn has_capability(&self, capability: TypeId) -> bool {
+        has_capability::<dyn VarBinExportable>(self, capability)
+    }
+
     fn append_to_builder(
         array: ArrayView<'_, Self>,
         builder: &mut dyn ArrayBuilder,

encodings/onpair/src/array.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::any::TypeId;
 use std::fmt::Debug;
 use std::fmt::Display;
 use std::fmt::Formatter;
@@ -40,7 +41,9 @@ use vortex_array::serde::ArrayChildren;
 use vortex_array::validity::Validity;
 use vortex_array::vtable::VTable;
 use vortex_array::vtable::ValidityVTable;
+use vortex_array::vtable::VarBinExportable;
 use vortex_array::vtable::child_to_validity;
+use vortex_array::vtable::has_capability;
 use vortex_array::vtable::validity_to_child;
 use vortex_buffer::Buffer;
 use vortex_buffer::ByteBuffer;
@@ -415,6 +418,8 @@ fn validate_parts(
     Ok(())
 }
 
+impl VarBinExportable for OnPair {}
+
 impl VTable for OnPair {
     type TypedArrayData = OnPairData;
     type OperationsVTable = Self;
@@ -586,6 +591,10 @@ impl VTable for OnPair {
         canonicalize_onpair(array.as_view(), ctx).map(ExecutionResult::done)
     }
 
+    fn has_capability(&self, capability: TypeId) -> bool {
+        has_capability::<dyn VarBinExportable>(self, capability)
+    }
+
     fn append_to_builder(
         array: ArrayView<'_, Self>,
         builder: &mut dyn ArrayBuilder,

encodings/sparse/src/lib.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::any::TypeId;
 use std::fmt::Debug;
 use std::fmt::Display;
 use std::fmt::Formatter;
@@ -47,6 +48,8 @@ use vortex_array::serde::ArrayChildren;
 use vortex_array::validity::Validity;
 use vortex_array::vtable::VTable;
 use vortex_array::vtable::ValidityVTable;
+use vortex_array::vtable::VarBinExportable;
+use vortex_array::vtable::has_capability;
 use vortex_buffer::Buffer;
 use vortex_buffer::ByteBufferMut;
 use vortex_error::VortexExpect as _;
@@ -187,6 +190,8 @@ impl ArrayEq for SparseData {
     }
 }
 
+impl VarBinExportable for Sparse {}
+
 impl VTable for Sparse {
     type TypedArrayData = SparseData;
 
@@ -359,6 +364,10 @@ impl VTable for Sparse {
         execute_sparse(parts, ctx).map(ExecutionResult::done)
     }
 
+    fn has_capability(&self, capability: TypeId) -> bool {
+        has_capability::<dyn VarBinExportable>(self, capability)
+    }
+
     fn append_to_builder(
         array: ArrayView<'_, Self>,
         builder: &mut dyn ArrayBuilder,

encodings/zstd/src/array.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::any::TypeId;
 use std::fmt::Debug;
 use std::fmt::Display;
 use std::fmt::Formatter;
@@ -45,7 +46,9 @@ use vortex_array::validity::Validity;
 use vortex_array::vtable::OperationsVTable;
 use vortex_array::vtable::VTable;
 use vortex_array::vtable::ValidityVTable;
+use vortex_array::vtable::VarBinExportable;
 use vortex_array::vtable::child_to_validity;
+use vortex_array::vtable::has_capability;
 use vortex_array::vtable::validity_to_child;
 use vortex_buffer::Alignment;
 use vortex_buffer::Buffer;
@@ -136,6 +139,8 @@ impl ArrayEq for ZstdData {
     }
 }
 
+impl VarBinExportable for Zstd {}
+
 impl VTable for Zstd {
     type TypedArrayData = ZstdData;
 
@@ -279,6 +284,10 @@ impl VTable for Zstd {
             .map(ExecutionResult::done)
     }
 
+    fn has_capability(&self, capability: TypeId) -> bool {
+        has_capability::<dyn VarBinExportable>(self, capability)
+    }
+
     fn append_to_builder(
         array: ArrayView<'_, Self>,
         builder: &mut dyn ArrayBuilder,

vortex-array/src/array/erased.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::any::TypeId;
 use std::any::type_name;
 use std::fmt::Debug;
 use std::fmt::Formatter;
@@ -385,19 +386,19 @@ impl ArrayRef {
 
     /// Does the array match the given matcher.
     #[inline]
-    pub fn is<M: Matcher>(&self) -> bool {
+    pub fn is<M: Matcher + ?Sized>(&self) -> bool {
         M::matches(self)
     }
 
     /// Returns the array downcast by the given matcher.
     #[inline]
-    pub fn as_<M: Matcher>(&self) -> M::Match<'_> {
+    pub fn as_<M: Matcher + ?Sized>(&self) -> M::Match<'_> {
         self.as_opt::<M>().vortex_expect("Failed to downcast")
     }
 
     /// Returns the array downcast by the given matcher.
     #[inline]
-    pub fn as_opt<M: Matcher>(&self) -> Option<M::Match<'_>> {
+    pub fn as_opt<M: Matcher + ?Sized>(&self) -> Option<M::Match<'_>> {
         M::try_match(self)
     }
 
@@ -443,6 +444,13 @@ impl ArrayRef {
         self.is::<AnyCanonical>()
     }
 
+    /// Whether this array's encoding implements the capability trait `C`.
+    ///
+    /// `false` unless the encoding reports `C` from [`VTable::has_capability`].
+    pub fn has_capability<C: ?Sized + 'static>(&self) -> bool {
+        self.0.data.has_capability(TypeId::of::<C>())
+    }
+
     /// Returns a new array with the slot at `slot_idx` replaced by `replacement`.
     ///
     /// This is only valid for physical rewrites: the replacement must have the same logical

vortex-array/src/array/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::any::Any;
+use std::any::TypeId;
 use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::hash::Hasher;
@@ -135,6 +136,10 @@ pub(crate) trait DynArrayData: 'static + private::Sealed + Send + Sync + Debug {
         ctx: &mut ExecutionCtx,
     ) -> VortexResult<()>;
 
+    /// Whether this array's encoding implements the given capability trait. See
+    /// [`VTable::has_capability`].
+    fn has_capability(&self, capability: TypeId) -> bool;
+
     // --- Visitor methods (formerly in ArrayVisitor) ---
 
     /// Returns the buffers of the array.
@@ -307,6 +312,10 @@ impl<V: VTable> DynArrayData for ArrayData<V> {
         Ok(())
     }
 
+    fn has_capability(&self, capability: TypeId) -> bool {
+        V::has_capability(&self.vtable, capability)
+    }
+
     fn buffers(&self, this: &ArrayRef) -> Vec<ByteBuffer> {
         let view = unsafe { ArrayView::new_unchecked(this, &self.data) };
         (0..V::nbuffers(view))

vortex-array/src/array/vtable/capability.rs

@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::any::TypeId;
+
+/// Whether `capability` names the capability trait `C`.
+///
+/// Called once per capability from [`VTable::has_capability`](super::VTable::has_capability).
+/// Passing the vtable as `&C` is what proves it implements `C`, so a vtable cannot claim a
+/// capability it does not have.
+pub fn has_capability<C: ?Sized + 'static>(_vtable: &C, capability: TypeId) -> bool {
+    capability == TypeId::of::<C>()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    trait Greet {}
+
+    trait Absent {}
+
+    struct Encoding;
+
+    impl Greet for Encoding {}
+
+    #[test]
+    fn has_capability_matches_only_the_requested_trait() {
+        assert!(has_capability::<dyn Greet>(
+            &Encoding,
+            TypeId::of::<dyn Greet>()
+        ));
+        assert!(!has_capability::<dyn Greet>(
+            &Encoding,
+            TypeId::of::<dyn Absent>()
+        ));
+    }
+}

vortex-array/src/array/vtable/mod.rs

@@ -11,16 +11,21 @@
 //! into these traits. Implementations should focus on encoding-specific work and uphold the
 //! documented postconditions.
 
+mod capability;
 mod operations;
 mod validity;
+mod varbin_exportable;
 
+use std::any::TypeId;
 use std::fmt::Debug;
 use std::fmt::Display;
 use std::fmt::Formatter;
 use std::hash::Hasher;
 
+pub use capability::*;
 pub use operations::*;
 pub use validity::*;
+pub use varbin_exportable::*;
 use vortex_error::VortexExpect;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
@@ -140,10 +145,28 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug {
         session: &VortexSession,
     ) -> VortexResult<ArrayParts<Self>>;
 
+    /// Whether this encoding implements the capability trait identified by `capability`.
+    ///
+    /// Capabilities are optional interfaces, queried through
+    /// [`ArrayRef::has_capability`](crate::ArrayRef::has_capability). Report each one with
+    /// [`has_capability`]:
+    ///
+    /// ```ignore
+    /// fn has_capability(&self, capability: TypeId) -> bool {
+    ///     has_capability::<dyn SomeCapability>(self, capability)
+    ///         || has_capability::<dyn AnotherCapability>(self, capability)
+    /// }
+    /// ```
+    fn has_capability(&self, capability: TypeId) -> bool {
+        _ = capability;
+        false
+    }
+
     /// Writes the array's logical values into a canonical builder.
     ///
     /// The default implementation executes the full array to [`Canonical`] and appends that result.
-    /// Encodings may override this to avoid materializing an intermediate canonical array.
+    /// Encodings may override this to avoid materializing an intermediate canonical array. An
+    /// override targeting a `VarBinBuilder` should also claim [`VarBinExportable`].
     fn append_to_builder(
         array: ArrayView<'_, Self>,
         builder: &mut dyn ArrayBuilder,

vortex-array/src/array/vtable/varbin_exportable.rs

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use crate::ArrayRef;
+use crate::matcher::Matcher;
+
+/// Capability for encodings whose [`append_to_builder`](super::VTable::append_to_builder) writes
+/// `Utf8`/`Binary` values straight into a [`VarBinBuilder`](crate::builders::VarBinBuilder).
+///
+/// Callers filling a `VarBinBuilder` should `execute_until::<dyn VarBinExportable>`: executing past
+/// these encodings reaches a canonical `VarBinView` that the builder has to re-lay out.
+///
+/// Claim it only where stopping is the better trade. `Dict` does not: it appends canonical values,
+/// but executing through it lets kernels such as FSST's `Dict` parent kernel decode through the
+/// dictionary instead.
+pub trait VarBinExportable: 'static + Send + Sync {}
+
+/// Matches every array whose encoding offers [`VarBinExportable`].
+impl Matcher for dyn VarBinExportable {
+    type Match<'a> = &'a ArrayRef;
+
+    #[inline]
+    fn try_match(array: &ArrayRef) -> Option<Self::Match<'_>> {
+        array
+            .has_capability::<dyn VarBinExportable>()
+            .then_some(array)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use vortex_error::VortexResult;
+
+    use super::*;
+    use crate::IntoArray;
+    use crate::arrays::DictArray;
+    use crate::arrays::PrimitiveArray;
+    use crate::arrays::VarBinViewArray;
+
+    #[test]
+    fn encodings_report_the_capability_through_their_vtable() -> VortexResult<()> {
+        let values = VarBinViewArray::from_iter_str(["a", "b"]).into_array();
+        assert!(values.has_capability::<dyn VarBinExportable>());
+        assert!(values.is::<dyn VarBinExportable>());
+
+        // Dict gathers canonical values into the builder but deliberately does not claim the
+        // capability, so execution continues through it.
+        let codes = PrimitiveArray::from_iter([0u8, 1, 0]).into_array();
+        let dict = DictArray::try_new(codes, values)?.into_array();
+        assert!(!dict.has_capability::<dyn VarBinExportable>());
+        Ok(())
+    }
+}