Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/paimon/src/spec/core_options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -676,8 +676,8 @@ impl<'a> CoreOptions<'a> {
/// default 32). Used as the per-operation fan-out limit for sorted BTree and
/// bitmap shard reads, global-index vector search, and primary-key vector
/// search. A value of `1` reproduces strict sequential execution. A
/// non-positive value is a misconfiguration and fails loud rather than being
/// silently clamped.
/// non-positive value, or one above [`MAX_GLOBAL_INDEX_THREAD_NUM`], is a
/// misconfiguration and fails loud rather than being silently clamped.
pub fn global_index_thread_num(&self) -> crate::Result<usize> {
let value = self
.parse_i64_option(GLOBAL_INDEX_THREAD_NUM_OPTION)?
Expand Down
45 changes: 44 additions & 1 deletion crates/paimon/src/table/format_table_scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,18 @@ fn parse_partition_date(value: &str) -> Option<i32> {
.ok()
}

fn supported_format_table_formats() -> Vec<&'static str> {
vec![
"parquet",
"orc",
"avro",
"row",
"mosaic",
#[cfg(feature = "vortex")]
"vortex",
]
}

fn supported_format_table_extension(format: &str) -> crate::Result<&'static str> {
match format.to_ascii_lowercase().as_str() {
"parquet" => Ok(".parquet"),
Expand All @@ -626,7 +638,9 @@ fn supported_format_table_extension(format: &str) -> crate::Result<&'static str>
"vortex" => Ok(".vortex"),
other => Err(crate::Error::Unsupported {
message: format!(
"Format table file.format '{other}' is not supported by the Rust reader yet"
"Format table file.format '{other}' is not supported by the Rust reader yet, \
expected one of: {}",
supported_format_table_formats().join(", ")
),
}),
}
Expand Down Expand Up @@ -656,3 +670,32 @@ fn data_file_meta(file_name: String, file_size: i64, schema_id: i64) -> DataFile
write_cols: None,
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_unsupported_format_lists_the_supported_ones() {
let error = supported_format_table_extension("csv").unwrap_err();
let crate::Error::Unsupported { message } = error else {
panic!("expected Unsupported, got {error:?}");
};
assert!(message.contains("'csv'"), "{message}");
for format in supported_format_table_formats() {
assert!(message.contains(format), "{format} missing from {message}");
}
}

#[test]
fn test_supported_formats_are_accepted_case_insensitively() {
for format in supported_format_table_formats() {
let expected = format!(".{format}");
assert_eq!(supported_format_table_extension(format).unwrap(), expected);
assert_eq!(
supported_format_table_extension(&format.to_ascii_uppercase()).unwrap(),
expected
);
}
}
}
2 changes: 1 addition & 1 deletion docs/src/sql.md
Original file line number Diff line number Diff line change
Expand Up @@ -2029,7 +2029,7 @@ deletion vectors enabled.
| `btree-index.fallback-scan-max-size` | `256mb` | Maximum total size of selected BTree global-index files for fallback scans used by range/between and suffix/contains/complex LIKE predicates; `0` disables BTree fallback index scans. |
| `bitmap-index.fallback-scan-max-size` | `256mb` | Maximum total size of selected bitmap global-index files for fallback scans used by range/between and suffix/contains/complex LIKE predicates; `0` disables bitmap fallback index scans. |
| `global-index.search-mode` | `fast` | Global index coverage mode for reads: `fast`, `full`, or `detail`. |
| `global-index.thread-num` | `32` | Number of threads used to search global index fields concurrently; must be greater than 0. |
| `global-index.thread-num` | `32` | Number of threads used to search global index fields concurrently; must be greater than 0 and must not exceed the runtime's task limit. |
| `global-index.column-update-action` | `THROW_ERROR` | What a commit does when it updates an indexed column: `THROW_ERROR` rejects the commit, `DROP_PARTITION_INDEX` drops the affected partition index instead. |

### Variant Shredding Options
Expand Down
Loading