From b45696cb1dbb380d2853ce0e0c208ee204d09a90 Mon Sep 17 00:00:00 2001 From: Reza Shahwaz Date: Fri, 30 Oct 2026 17:05:39 -0500 Subject: [PATCH 1/4] fix: return an error instead of aborting in `repeat` for huge LargeUtf8 results `repeat` capped LargeUtf8 results at i64::MAX bytes, which admits sizes no allocation can satisfy, so the process aborted or panicked instead of returning an error. Cap LargeUtf8 results at u32::MAX bytes in both the scalar and array paths. Closes #26948 --- datafusion/functions/src/string/repeat.rs | 39 +++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs index a7a09e2f767fa..baa5d08498272 100644 --- a/datafusion/functions/src/string/repeat.rs +++ b/datafusion/functions/src/string/repeat.rs @@ -22,6 +22,14 @@ use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; use datafusion_expr_common::sort_properties::ExprProperties; use datafusion_macros::user_doc; +/// Largest result, in bytes, that `repeat` produces for `LargeUtf8`. +/// +/// `LargeUtf8` offsets would allow up to `i64::MAX` bytes, but the result is +/// allocated up front, and an allocation larger than the address space aborts +/// the process instead of returning an error. This cap matches DuckDB's string +/// size limit and is twice the `Utf8` limit of `i32::MAX`. +const LARGE_UTF8_MAX_SIZE: usize = u32::MAX as usize; + #[user_doc( doc_section(label = "String Functions"), description = "Returns a string with an input string repeated a specified number.", @@ -125,7 +133,7 @@ impl ScalarUDFImpl for RepeatFunc { compute_repeat(s, count, i32::MAX as usize)?, )), ScalarValue::LargeUtf8(Some(s)) => ScalarValue::LargeUtf8(Some( - compute_repeat(s, count, i64::MAX as usize)?, + compute_repeat(s, count, LARGE_UTF8_MAX_SIZE)?, )), _ => { return internal_err!( @@ -225,7 +233,7 @@ fn repeat(string_array: &ArrayRef, count_array: &ArrayRef) -> Result { LargeUtf8 => { let string_arr = string_array.as_string::(); let (total_capacity, max_item_capacity) = - calculate_capacities(&string_arr, number_array, i64::MAX as usize)?; + calculate_capacities(&string_arr, number_array, LARGE_UTF8_MAX_SIZE)?; let builder = GenericStringArrayBuilder::::with_capacity( string_array.len(), total_capacity, @@ -454,6 +462,21 @@ mod tests { Utf8, StringArray ); + test_function!( + RepeatFunc::new(), + vec![ + ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(String::from("a")))), + ColumnarValue::Scalar(ScalarValue::Int64(Some(1 << 60))), + ], + exec_err!( + "string size overflow on repeat, max size is {}, but got {}", + u32::MAX, + 1usize << 60 + ), + &str, + LargeUtf8, + LargeStringArray + ); Ok(()) } @@ -519,6 +542,18 @@ mod tests { ); } + #[test] + fn test_repeat_large_string_array_exceeds_max_size() { + let strings: ArrayRef = Arc::new(LargeStringArray::from(vec![Some("a")])); + let counts: ArrayRef = Arc::new(Int64Array::from(vec![Some(1 << 60)])); + + let err = super::repeat(&strings, &counts).unwrap_err().to_string(); + assert!( + err.contains("string size overflow on repeat"), + "unexpected error: {err}" + ); + } + #[test] fn test_repeat_sliced_large_string_with_null_offset() { let (strings, counts) = From f7c7126e2f29e02e769fc4006ce980d29ac1bb5c Mon Sep 17 00:00:00 2001 From: Reza Shahwaz Date: Fri, 30 Oct 2026 19:45:34 -0500 Subject: [PATCH 2/4] fix: also cap the total LargeUtf8 output size in the repeat array path The per-row cap alone still let many rows add up to an allocation no system can satisfy, which panicked in the Arrow buffer instead of returning an error. --- datafusion/functions/src/string/repeat.rs | 31 +++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs index baa5d08498272..feac6bb2196b5 100644 --- a/datafusion/functions/src/string/repeat.rs +++ b/datafusion/functions/src/string/repeat.rs @@ -22,12 +22,13 @@ use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; use datafusion_expr_common::sort_properties::ExprProperties; use datafusion_macros::user_doc; -/// Largest result, in bytes, that `repeat` produces for `LargeUtf8`. +/// Largest result, in bytes, that `repeat` produces for `LargeUtf8`, both for a +/// single value and for the total of an array. /// /// `LargeUtf8` offsets would allow up to `i64::MAX` bytes, but the result is /// allocated up front, and an allocation larger than the address space aborts /// the process instead of returning an error. This cap matches DuckDB's string -/// size limit and is twice the `Utf8` limit of `i32::MAX`. +/// size limit and is about twice the `Utf8` limit of `i32::MAX`. const LARGE_UTF8_MAX_SIZE: usize = u32::MAX as usize; #[user_doc( @@ -234,6 +235,15 @@ fn repeat(string_array: &ArrayRef, count_array: &ArrayRef) -> Result { let string_arr = string_array.as_string::(); let (total_capacity, max_item_capacity) = calculate_capacities(&string_arr, number_array, LARGE_UTF8_MAX_SIZE)?; + // The builder allocates the whole output up front, so the cap must + // also bound the total, not only each row. + if total_capacity > LARGE_UTF8_MAX_SIZE { + return exec_err!( + "string size overflow on repeat, max size is {}, but got {}", + LARGE_UTF8_MAX_SIZE, + total_capacity + ); + } let builder = GenericStringArrayBuilder::::with_capacity( string_array.len(), total_capacity, @@ -554,6 +564,23 @@ mod tests { ); } + #[test] + fn test_repeat_large_string_array_total_exceeds_max_size() { + // Each row is under the cap, but together they exceed it. + let strings: ArrayRef = + Arc::new(LargeStringArray::from(vec![Some("a"), Some("a")])); + let counts: ArrayRef = Arc::new(Int64Array::from(vec![ + Some(3_000_000_000), + Some(3_000_000_000), + ])); + + let err = super::repeat(&strings, &counts).unwrap_err().to_string(); + assert!( + err.contains("string size overflow on repeat"), + "unexpected error: {err}" + ); + } + #[test] fn test_repeat_sliced_large_string_with_null_offset() { let (strings, counts) = From 8ca4139386c5ae9a78bc8528c8df3cfc2a74e9c7 Mon Sep 17 00:00:00 2001 From: Reza Shahwaz Date: Thu, 5 Nov 2026 14:38:02 -0600 Subject: [PATCH 3/4] Trim LARGE_UTF8_MAX_SIZE comment and fold the LargeUtf8 total tests Co-authored-by: Matt Butrovich <6855576+mbutrovich@users.noreply.github.com> --- datafusion/functions/src/string/repeat.rs | 43 ++++++++--------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs index feac6bb2196b5..847a9ba2c34bb 100644 --- a/datafusion/functions/src/string/repeat.rs +++ b/datafusion/functions/src/string/repeat.rs @@ -23,12 +23,9 @@ use datafusion_expr_common::sort_properties::ExprProperties; use datafusion_macros::user_doc; /// Largest result, in bytes, that `repeat` produces for `LargeUtf8`, both for a -/// single value and for the total of an array. -/// -/// `LargeUtf8` offsets would allow up to `i64::MAX` bytes, but the result is -/// allocated up front, and an allocation larger than the address space aborts -/// the process instead of returning an error. This cap matches DuckDB's string -/// size limit and is about twice the `Utf8` limit of `i32::MAX`. +/// single value and for the total of an array. The result is allocated up front, +/// and an allocation the allocator refuses aborts or panics instead of returning +/// an error. const LARGE_UTF8_MAX_SIZE: usize = u32::MAX as usize; #[user_doc( @@ -552,32 +549,22 @@ mod tests { ); } - #[test] - fn test_repeat_large_string_array_exceeds_max_size() { - let strings: ArrayRef = Arc::new(LargeStringArray::from(vec![Some("a")])); - let counts: ArrayRef = Arc::new(Int64Array::from(vec![Some(1 << 60)])); - - let err = super::repeat(&strings, &counts).unwrap_err().to_string(); - assert!( - err.contains("string size overflow on repeat"), - "unexpected error: {err}" - ); - } - #[test] fn test_repeat_large_string_array_total_exceeds_max_size() { - // Each row is under the cap, but together they exceed it. - let strings: ArrayRef = - Arc::new(LargeStringArray::from(vec![Some("a"), Some("a")])); - let counts: ArrayRef = Arc::new(Int64Array::from(vec![ - Some(3_000_000_000), - Some(3_000_000_000), - ])); + // Each row is at the cap, but the total is larger than any allocation + // can satisfy, so a missing total check fails at once. + let n = 1 << 17; + let strings: ArrayRef = Arc::new(LargeStringArray::from(vec![Some("a"); n])); + let counts: ArrayRef = Arc::new(Int64Array::from(vec![Some(u32::MAX as i64); n])); let err = super::repeat(&strings, &counts).unwrap_err().to_string(); - assert!( - err.contains("string size overflow on repeat"), - "unexpected error: {err}" + assert_eq!( + err, + format!( + "Execution error: string size overflow on repeat, max size is {}, but got {}", + u32::MAX, + n * u32::MAX as usize + ) ); } From 365f3440c2050ad970eeb89b8fdc78f583b5b84a Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Thu, 5 Nov 2026 16:51:04 -0500 Subject: [PATCH 4/4] Update datafusion/functions/src/string/repeat.rs --- datafusion/functions/src/string/repeat.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs index 847a9ba2c34bb..66386e9672d48 100644 --- a/datafusion/functions/src/string/repeat.rs +++ b/datafusion/functions/src/string/repeat.rs @@ -557,7 +557,9 @@ mod tests { let strings: ArrayRef = Arc::new(LargeStringArray::from(vec![Some("a"); n])); let counts: ArrayRef = Arc::new(Int64Array::from(vec![Some(u32::MAX as i64); n])); - let err = super::repeat(&strings, &counts).unwrap_err().to_string(); + let err = super::repeat(&strings, &counts) + .unwrap_err() + .strip_backtrace(); assert_eq!( err, format!(