fix(index): convert JSON values during index updates

rust/lance-index-core/src/scalar.rs

@@ -4,7 +4,7 @@
 //! Abstract scalar index traits and types for Lance index plugins
 
 use arrow_array::{BooleanArray, RecordBatch, UInt64Array};
-use arrow_schema::Schema;
+use arrow_schema::{DataType, Schema};
 use async_trait::async_trait;
 use bytes::Bytes;
 use datafusion::physical_plan::SendableRecordBatchStream;
@@ -555,6 +555,16 @@ pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf {
     /// with the same configuration on another dataset.
     fn derive_index_params(&self) -> Result<ScalarIndexParams>;
 
+    /// Returns the value type expected by [`Self::update`], when the index has
+    /// a durable type contract for its training data.
+    ///
+    /// Wrapper indices use this to transform new data to the same type as the
+    /// loaded index instead of inferring a potentially different type from an
+    /// update batch. Index types without such a contract may return `None`.
+    fn training_data_type(&self) -> Option<DataType> {
+        None
+    }
+
     /// Global `[min, max]` of the indexed column from index metadata, without a
     /// scan, or `None` if this index type cannot supply a sound bound. When
     /// `Some`, the range is a superset of live values (conservative under

rust/lance-index/src/scalar/btree.rs

@@ -2368,6 +2368,10 @@ impl ScalarIndex for BTreeIndex {
         })?;
         Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::BTree).with_params(&params))
     }
+
+    fn training_data_type(&self) -> Option<DataType> {
+        Some(self.data_type.clone())
+    }
 }
 
 struct BatchStats {