diff --git a/crates/paimon/src/spec/mod.rs b/crates/paimon/src/spec/mod.rs index 5fe8faa11..97be205c9 100644 --- a/crates/paimon/src/spec/mod.rs +++ b/crates/paimon/src/spec/mod.rs @@ -104,8 +104,8 @@ mod partition; pub use partition::Partition; mod partition_utils; pub(crate) use partition_utils::{ - bucket_path, bucket_path_under, data_file_path, escape_path_name, relative_bucket_path, - unescape_path_name, PartitionComputer, + bucket_path, bucket_path_under, data_file_path, escape_path_name, is_java_whitespace_only, + relative_bucket_path, unescape_path_name, PartitionComputer, }; mod predicate; pub(crate) use predicate::datum_cmp; diff --git a/crates/paimon/src/spec/partition_utils.rs b/crates/paimon/src/spec/partition_utils.rs index 7c3dda8c1..3211de116 100644 --- a/crates/paimon/src/spec/partition_utils.rs +++ b/crates/paimon/src/spec/partition_utils.rs @@ -393,7 +393,12 @@ fn format_partition_value( DataType::Char(_) | DataType::VarChar(_) => { let s = row.get_string(pos)?; - if s.trim().is_empty() { + // Java folds a partition value to the default name when + // `StringUtils.isNullOrWhitespaceOnly` holds. Rust `str::trim` uses + // a different whitespace set (e.g. it trims NBSP / U+2007 / U+202F, + // which Java does not, and keeps U+001C-U+001F, which Java trims), + // so reuse the Java-matching predicate the Binary arm already uses. + if is_java_whitespace_only(s) { return Ok(default_partition_name.to_string()); } s.to_string() @@ -519,7 +524,7 @@ fn decode_java_utf8(mut bytes: &[u8]) -> String { /// Java `StringUtils.isNullOrWhitespaceOnly` checks each UTF-16 code unit with /// `Character.isWhitespace`; its whitespace set differs from Rust `str::trim`. -fn is_java_whitespace_only(value: &str) -> bool { +pub(crate) fn is_java_whitespace_only(value: &str) -> bool { value.chars().all(|ch| { matches!( ch, @@ -1169,6 +1174,35 @@ mod tests { ); } + #[test] + fn test_string_partition_whitespace_matches_java() { + // U+001C (file separator) is whitespace to Java's `Character.isWhitespace` + // but not to Rust `str::trim`; Java folds such a value to the default + // partition name, so we must too. + assert_single_partition( + "dt", + DataType::VarChar(VarCharType::default()), + |b| b.write_string(0, "\u{001C}"), + "dt=__DEFAULT_PARTITION__/", + true, + ); + + // A non-breaking space (U+00A0) is whitespace to Rust `str::trim` but not + // to Java, so Java keeps it as the partition value rather than folding it. + let fields = vec![make_field("dt", DataType::VarChar(VarCharType::default()))]; + let keys = vec!["dt".to_string()]; + let computer = + PartitionComputer::new(&keys, &fields, TEST_DEFAULT_PARTITION_NAME, true).unwrap(); + let mut builder = TestRowBuilder::new(1); + builder.write_string(0, "\u{00A0}"); + let row = builder.build(); + let result = computer.generate_partition_path(&row).unwrap(); + assert_ne!( + result, "dt=__DEFAULT_PARTITION__/", + "a non-breaking space must not fold to the default partition (Java keeps it)" + ); + } + #[test] fn test_boolean_partition() { assert_single_partition( diff --git a/crates/paimon/src/table/format_partition.rs b/crates/paimon/src/table/format_partition.rs index 748c8b284..4a31b2c93 100644 --- a/crates/paimon/src/table/format_partition.rs +++ b/crates/paimon/src/table/format_partition.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use chrono::NaiveDate; use crate::io::FileIO; -use crate::spec::{escape_path_name, unescape_path_name, DataType, Datum}; +use crate::spec::{escape_path_name, is_java_whitespace_only, unescape_path_name, DataType, Datum}; const UNIX_EPOCH_DAYS_FROM_CE: i32 = 719_163; @@ -78,7 +78,13 @@ impl FormatTablePartitionPaths { } let mut segments = Vec::with_capacity(leading_values.len()); for (key, value) in self.partition_keys.iter().zip(leading_values) { - if value.trim().is_empty() { + // A value that folds to the default partition name (Java + // `StringUtils.isNullOrWhitespaceOnly`) is not stored under its + // literal spelling, so a prefix pattern built from it would miss the + // data; skip pushdown and let the caller list every partition. + // Mirrors Java `buildPartitionNamePrefixPattern`, which returns null + // for such a value; `str::trim` folds a different whitespace set. + if is_java_whitespace_only(value) { return None; } segments.push(format!( @@ -309,7 +315,11 @@ pub fn format_partition_value( (Datum::Int(value), DataType::Int(_)) => Some(value.to_string()), (Datum::Long(value), DataType::BigInt(_)) => Some(value.to_string()), (Datum::String(value), DataType::Char(_) | DataType::VarChar(_)) => { - if value.trim().is_empty() { + // Fold to the default partition name exactly when Java + // `InternalRowPartitionComputer` does (`isNullOrWhitespaceOnly`), so + // the directory matches cross-engine; `str::trim` uses a different + // whitespace set (folds NBSP, keeps U+001C-U+001F). + if is_java_whitespace_only(value) { Some(default_partition_name.to_string()) } else { Some(value.clone()) @@ -378,7 +388,7 @@ fn last_path_segment(path: &str) -> Option<&str> { #[cfg(test)] mod tests { use super::*; - use crate::spec::{BooleanType, DateType}; + use crate::spec::{BooleanType, DateType, VarCharType}; #[test] fn test_parse_format_partition_value() { @@ -524,4 +534,46 @@ mod tests { assert_eq!(is_storage_not_found(&error), expected); } } + + #[test] + fn test_format_partition_value_folds_java_whitespace_only() { + let varchar = DataType::VarChar(VarCharType::string_type()); + let default = "__DEFAULT_PARTITION__"; + + // U+00A0 (non-breaking space) is not `Character.isWhitespace` in Java, so + // it stays a real partition value. `str::trim` would wrongly fold it to + // the default and diverge from a Java-written directory. + assert_eq!( + format_partition_value( + &Datum::String("\u{00A0}".to_string()), + &varchar, + default, + false + ), + Some("\u{00A0}".to_string()) + ); + + // U+001C (file separator) is `Character.isWhitespace` in Java, so it folds + // to the default. `str::trim` keeps it, which would produce a literal + // directory Java never writes. + assert_eq!( + format_partition_value( + &Datum::String("\u{001C}".to_string()), + &varchar, + default, + false + ), + Some(default.to_string()) + ); + + // An ASCII-blank value folds under both rules; a normal value is kept. + assert_eq!( + format_partition_value(&Datum::String(" ".to_string()), &varchar, default, false), + Some(default.to_string()) + ); + assert_eq!( + format_partition_value(&Datum::String("dt".to_string()), &varchar, default, false), + Some("dt".to_string()) + ); + } }