From 1f8333a3b17db7bea930b738a15e3cf29cb45ebe Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Mon, 28 Sep 2026 22:03:27 +0800 Subject: [PATCH 1/2] fix(spec): match Java whitespace for CHAR/VARCHAR partition values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CHAR/VARCHAR arm of the partition-value computer folded a value to the default partition name when `str::trim` left it empty, but Rust's whitespace set differs from Java's `Character.isWhitespace`: `str::trim` strips a non-breaking space (U+00A0) and U+2007 / U+202F, which Java keeps, and keeps U+001C-U+001F, which Java strips. Because the partition value is recomputed on both the write and read paths, such a value produced a different partition directory in Rust than in Java, so cross-engine readers miss the data. Reuse `is_java_whitespace_only` — already used by the Binary arm — so the string arm folds exactly what Java folds. --- crates/paimon/src/spec/partition_utils.rs | 36 ++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/crates/paimon/src/spec/partition_utils.rs b/crates/paimon/src/spec/partition_utils.rs index 7c3dda8c1..4d5d498b1 100644 --- a/crates/paimon/src/spec/partition_utils.rs +++ b/crates/paimon/src/spec/partition_utils.rs @@ -393,7 +393,12 @@ fn format_partition_value( DataType::Char(_) | DataType::VarChar(_) => { let s = row.get_string(pos)?; - if s.trim().is_empty() { + // Java folds a partition value to the default name when + // `StringUtils.isNullOrWhitespaceOnly` holds. Rust `str::trim` uses + // a different whitespace set (e.g. it trims NBSP / U+2007 / U+202F, + // which Java does not, and keeps U+001C-U+001F, which Java trims), + // so reuse the Java-matching predicate the Binary arm already uses. + if is_java_whitespace_only(s) { return Ok(default_partition_name.to_string()); } s.to_string() @@ -1169,6 +1174,35 @@ mod tests { ); } + #[test] + fn test_string_partition_whitespace_matches_java() { + // U+001C (file separator) is whitespace to Java's `Character.isWhitespace` + // but not to Rust `str::trim`; Java folds such a value to the default + // partition name, so we must too. + assert_single_partition( + "dt", + DataType::VarChar(VarCharType::default()), + |b| b.write_string(0, "\u{001C}"), + "dt=__DEFAULT_PARTITION__/", + true, + ); + + // A non-breaking space (U+00A0) is whitespace to Rust `str::trim` but not + // to Java, so Java keeps it as the partition value rather than folding it. + let fields = vec![make_field("dt", DataType::VarChar(VarCharType::default()))]; + let keys = vec!["dt".to_string()]; + let computer = + PartitionComputer::new(&keys, &fields, TEST_DEFAULT_PARTITION_NAME, true).unwrap(); + let mut builder = TestRowBuilder::new(1); + builder.write_string(0, "\u{00A0}"); + let row = builder.build(); + let result = computer.generate_partition_path(&row).unwrap(); + assert_ne!( + result, "dt=__DEFAULT_PARTITION__/", + "a non-breaking space must not fold to the default partition (Java keeps it)" + ); + } + #[test] fn test_boolean_partition() { assert_single_partition( From 85b3f917c0e0908cfaa5848a672ac273072b3339 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Wed, 30 Sep 2026 22:22:45 +0800 Subject: [PATCH 2/2] fix(table): match Java whitespace in Format Table partition paths `format_partition_value` folded a CHAR/VARCHAR value to the default partition with `str::trim().is_empty()`, whose whitespace set differs from Java's `Character.isWhitespace`: U+00A0 (which Java keeps) folded in Rust, and U+001C-U+001F (which Java folds) did not, so a Format Table partition directory diverged from a Java-written one on both the write and read paths. The `name_prefix_pattern` pushdown skip used the same `str::trim` check. Reuse `is_java_whitespace_only` (the predicate the spec-side partition computer already uses) in both, matching Java `InternalRowPartitionComputer` (value folding) and `PartitionPathUtils.buildPartitionNamePrefixPattern` (the pushdown skip). Add a partition-value test covering both divergent directions. --- crates/paimon/src/spec/mod.rs | 4 +- crates/paimon/src/spec/partition_utils.rs | 2 +- crates/paimon/src/table/format_partition.rs | 60 +++++++++++++++++++-- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/crates/paimon/src/spec/mod.rs b/crates/paimon/src/spec/mod.rs index 5fe8faa11..97be205c9 100644 --- a/crates/paimon/src/spec/mod.rs +++ b/crates/paimon/src/spec/mod.rs @@ -104,8 +104,8 @@ mod partition; pub use partition::Partition; mod partition_utils; pub(crate) use partition_utils::{ - bucket_path, bucket_path_under, data_file_path, escape_path_name, relative_bucket_path, - unescape_path_name, PartitionComputer, + bucket_path, bucket_path_under, data_file_path, escape_path_name, is_java_whitespace_only, + relative_bucket_path, unescape_path_name, PartitionComputer, }; mod predicate; pub(crate) use predicate::datum_cmp; diff --git a/crates/paimon/src/spec/partition_utils.rs b/crates/paimon/src/spec/partition_utils.rs index 4d5d498b1..3211de116 100644 --- a/crates/paimon/src/spec/partition_utils.rs +++ b/crates/paimon/src/spec/partition_utils.rs @@ -524,7 +524,7 @@ fn decode_java_utf8(mut bytes: &[u8]) -> String { /// Java `StringUtils.isNullOrWhitespaceOnly` checks each UTF-16 code unit with /// `Character.isWhitespace`; its whitespace set differs from Rust `str::trim`. -fn is_java_whitespace_only(value: &str) -> bool { +pub(crate) fn is_java_whitespace_only(value: &str) -> bool { value.chars().all(|ch| { matches!( ch, diff --git a/crates/paimon/src/table/format_partition.rs b/crates/paimon/src/table/format_partition.rs index 748c8b284..4a31b2c93 100644 --- a/crates/paimon/src/table/format_partition.rs +++ b/crates/paimon/src/table/format_partition.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use chrono::NaiveDate; use crate::io::FileIO; -use crate::spec::{escape_path_name, unescape_path_name, DataType, Datum}; +use crate::spec::{escape_path_name, is_java_whitespace_only, unescape_path_name, DataType, Datum}; const UNIX_EPOCH_DAYS_FROM_CE: i32 = 719_163; @@ -78,7 +78,13 @@ impl FormatTablePartitionPaths { } let mut segments = Vec::with_capacity(leading_values.len()); for (key, value) in self.partition_keys.iter().zip(leading_values) { - if value.trim().is_empty() { + // A value that folds to the default partition name (Java + // `StringUtils.isNullOrWhitespaceOnly`) is not stored under its + // literal spelling, so a prefix pattern built from it would miss the + // data; skip pushdown and let the caller list every partition. + // Mirrors Java `buildPartitionNamePrefixPattern`, which returns null + // for such a value; `str::trim` folds a different whitespace set. + if is_java_whitespace_only(value) { return None; } segments.push(format!( @@ -309,7 +315,11 @@ pub fn format_partition_value( (Datum::Int(value), DataType::Int(_)) => Some(value.to_string()), (Datum::Long(value), DataType::BigInt(_)) => Some(value.to_string()), (Datum::String(value), DataType::Char(_) | DataType::VarChar(_)) => { - if value.trim().is_empty() { + // Fold to the default partition name exactly when Java + // `InternalRowPartitionComputer` does (`isNullOrWhitespaceOnly`), so + // the directory matches cross-engine; `str::trim` uses a different + // whitespace set (folds NBSP, keeps U+001C-U+001F). + if is_java_whitespace_only(value) { Some(default_partition_name.to_string()) } else { Some(value.clone()) @@ -378,7 +388,7 @@ fn last_path_segment(path: &str) -> Option<&str> { #[cfg(test)] mod tests { use super::*; - use crate::spec::{BooleanType, DateType}; + use crate::spec::{BooleanType, DateType, VarCharType}; #[test] fn test_parse_format_partition_value() { @@ -524,4 +534,46 @@ mod tests { assert_eq!(is_storage_not_found(&error), expected); } } + + #[test] + fn test_format_partition_value_folds_java_whitespace_only() { + let varchar = DataType::VarChar(VarCharType::string_type()); + let default = "__DEFAULT_PARTITION__"; + + // U+00A0 (non-breaking space) is not `Character.isWhitespace` in Java, so + // it stays a real partition value. `str::trim` would wrongly fold it to + // the default and diverge from a Java-written directory. + assert_eq!( + format_partition_value( + &Datum::String("\u{00A0}".to_string()), + &varchar, + default, + false + ), + Some("\u{00A0}".to_string()) + ); + + // U+001C (file separator) is `Character.isWhitespace` in Java, so it folds + // to the default. `str::trim` keeps it, which would produce a literal + // directory Java never writes. + assert_eq!( + format_partition_value( + &Datum::String("\u{001C}".to_string()), + &varchar, + default, + false + ), + Some(default.to_string()) + ); + + // An ASCII-blank value folds under both rules; a normal value is kept. + assert_eq!( + format_partition_value(&Datum::String(" ".to_string()), &varchar, default, false), + Some(default.to_string()) + ); + assert_eq!( + format_partition_value(&Datum::String("dt".to_string()), &varchar, default, false), + Some("dt".to_string()) + ); + } }