diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md index badb49b3b84..876311e96cb 100644 --- a/docs/configuration/index-config.md +++ b/docs/configuration/index-config.md @@ -405,7 +405,7 @@ If, in addition, `attributes` is set as a default search field, then `color:red` #### `tie_breaker` type -The `tie_breaker` type is a field whose values are generated by Quickwit when a split is finalized. It behaves like a `u64` field with `fast: true`, `stored: false` and `indexed: false`, and these options cannot be changed. Use it as a secondary sort key to reduce ties among documents with equal primary sort values. +The `tie_breaker` type is a field whose values are generated by Quickwit when a split is finalized. It behaves like an `i64` field with `fast: true`, `stored: false` and `indexed: false`, and these options cannot be changed. Use it as a secondary sort key to reduce ties among documents with equal primary sort values. ```yaml name: tie_breaker diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index afea6ffa4c3..4c929de6fc0 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -4532,7 +4532,7 @@ dependencies = [ [[package]] name = "jitexpr" version = "0.1.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "cranelift", "cranelift-jit", @@ -5995,7 +5995,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "stable_deref_trait", ] @@ -10305,7 +10305,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" version = "0.27.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "aho-corasick", "arc-swap", @@ -10346,7 +10346,7 @@ dependencies = [ "tantivy-columnar", "tantivy-common", "tantivy-fst", - "tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38)", + "tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe)", "tantivy-sstable", "tantivy-stacker", "tantivy-tokenizer-api", @@ -10363,7 +10363,7 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.10.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "bitpacking", ] @@ -10371,7 +10371,7 @@ dependencies = [ [[package]] name = "tantivy-columnar" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "downcast-rs", "fastdivide", @@ -10387,7 +10387,7 @@ dependencies = [ [[package]] name = "tantivy-common" version = "0.11.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "async-trait", "byteorder", @@ -10423,7 +10423,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.26.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "fnv", "nom 7.1.3", @@ -10435,7 +10435,7 @@ dependencies = [ [[package]] name = "tantivy-sstable" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "futures-util", "itertools 0.14.0", @@ -10448,7 +10448,7 @@ dependencies = [ [[package]] name = "tantivy-stacker" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "murmurhash32", "tantivy-common", @@ -10457,7 +10457,7 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=4abc20f924e031a91f89c8cb44575e302e68fb38#4abc20f924e031a91f89c8cb44575e302e68fb38" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" dependencies = [ "serde", ] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index e8f90aad546..1ade0c4c0e2 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -378,7 +378,7 @@ quickwit-telemetry-exporters = { path = "quickwit-telemetry-exporters" } quickwit-transport = { path = "quickwit-transport" } # This is the tip of tantivy main. -tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "4abc20f924e031a91f89c8cb44575e302e68fb38", default-features = false, features = [ +tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "2e87c2eb0c8577c96c82a777f345bbf236a985fe", default-features = false, features = [ "jitexpr", "lz4-compression", "mmap", @@ -387,7 +387,7 @@ tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "4abc20f924e "columnar-zstd-compression", ] } # Must stay on the same rev as `tantivy`. -tantivy-common = { git = "https://github.com/quickwit-oss/tantivy/", rev = "4abc20f924e031a91f89c8cb44575e302e68fb38" } +tantivy-common = { git = "https://github.com/quickwit-oss/tantivy/", rev = "2e87c2eb0c8577c96c82a777f345bbf236a985fe" } tantivy-fst = "0.5" # This is actually not used directly the goal is to fix the version diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper/tantivy_val_to_json.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper/tantivy_val_to_json.rs index 191c700e1a1..4b8367e4333 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper/tantivy_val_to_json.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper/tantivy_val_to_json.rs @@ -244,8 +244,8 @@ pub fn formatted_tantivy_value_to_json( LeafType::F64(numeric_options) => value_to_float(value, numeric_options), LeafType::U64(numeric_options) => value_to_u64(value, numeric_options), LeafType::TieBreaker => { - // Only `output_format` is read by `value_to_u64`. The other options describe what a - // tie-breaker field is: a fast-only `u64` whose values are never parsed from documents. + // Only `output_format` is read by `value_to_i64`. The other options describe what a + // tie-breaker field is: a fast-only `i64` whose values are never parsed from documents. let numeric_options = QuickwitNumericOptions { description: None, stored: false, @@ -254,7 +254,7 @@ pub fn formatted_tantivy_value_to_json( coerce: false, output_format: NumericOutputFormat::Number, }; - value_to_u64(value, &numeric_options) + value_to_i64(value, &numeric_options) } LeafType::I64(numeric_options) => value_to_i64(value, numeric_options), LeafType::Json(_) => { @@ -411,6 +411,16 @@ mod tests { ); } + #[test] + fn test_tantivy_value_to_json_value_tie_breaker() { + for value in [i32::MIN as i64, -1, 0, i32::MAX as i64] { + assert_eq!( + formatted_tantivy_value_to_json(TantivyValue::I64(value), &LeafType::TieBreaker), + Some(serde_json::json!(value)) + ); + } + } + #[test] fn test_tantivy_value_to_json_value_u64() { let numeric_options_number = QuickwitNumericOptions::default(); diff --git a/quickwit/quickwit-query/src/query_ast/range_query.rs b/quickwit/quickwit-query/src/query_ast/range_query.rs index be0029a5890..4c21268cf1e 100644 --- a/quickwit/quickwit-query/src/query_ast/range_query.rs +++ b/quickwit/quickwit-query/src/query_ast/range_query.rs @@ -143,7 +143,7 @@ impl BuildTantivyAst for RangeQuery { ) .into() } - tantivy::schema::FieldType::U64(_) | tantivy::schema::FieldType::TieBreaker => { + tantivy::schema::FieldType::U64(_) => { let (lower_bound, upper_bound) = convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?; FastFieldRangeQuery::new( @@ -152,7 +152,7 @@ impl BuildTantivyAst for RangeQuery { ) .into() } - tantivy::schema::FieldType::I64(_) => { + tantivy::schema::FieldType::I64(_) | tantivy::schema::FieldType::TieBreaker => { let (lower_bound, upper_bound) = convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?; FastFieldRangeQuery::new( @@ -363,7 +363,7 @@ mod tests { let schema = schema_builder.build(); let range_query = RangeQuery { field: "tie_breaker".to_string(), - lower_bound: Bound::Included(JsonLiteral::String("1980".to_string())), + lower_bound: Bound::Included(JsonLiteral::String("-1980".to_string())), upper_bound: Bound::Included(JsonLiteral::String("1989".to_string())), }; let tantivy_ast = range_query @@ -374,7 +374,7 @@ mod tests { assert_eq!( format!("{leaf:?}"), "FastFieldRangeQuery { bounds: BoundsRange { lower_bound: Included(Term(field=0, \ - type=U64, 1980)), upper_bound: Included(Term(field=0, type=U64, 1989)) } }" + type=I64, -1980)), upper_bound: Included(Term(field=0, type=I64, 1989)) } }" ); } diff --git a/quickwit/quickwit-query/src/query_ast/term_query.rs b/quickwit/quickwit-query/src/query_ast/term_query.rs index 28dfe53fe97..d0796b71ccc 100644 --- a/quickwit/quickwit-query/src/query_ast/term_query.rs +++ b/quickwit/quickwit-query/src/query_ast/term_query.rs @@ -220,7 +220,7 @@ mod tests { fn test_term_query_with_tie_breaker() { let term_query = TermQuery { field: "tie_breaker".to_string(), - value: "42".to_string(), + value: "-42".to_string(), }; let mut schema_builder = Schema::builder(); schema_builder.add_tie_breaker_field("tie_breaker"); @@ -231,7 +231,7 @@ mod tests { let leaf = tantivy_query_ast.as_leaf().unwrap(); assert_eq!( &format!("{leaf:?}"), - "TermQuery(Term(field=0, type=U64, 42))" + "TermQuery(Term(field=0, type=I64, -42))" ); } } diff --git a/quickwit/quickwit-query/src/query_ast/utils.rs b/quickwit/quickwit-query/src/query_ast/utils.rs index 90806465f23..6c930916dc9 100644 --- a/quickwit/quickwit-query/src/query_ast/utils.rs +++ b/quickwit/quickwit-query/src/query_ast/utils.rs @@ -121,12 +121,12 @@ fn compute_query_with_field( ) -> Result { let field_type = field_entry.field_type(); match field_type { - FieldType::U64(_) | FieldType::TieBreaker => { + FieldType::U64(_) => { let val = parse_value_from_user_text::(value, field_entry.name())?; let term = Term::from_field_u64(field, val); Ok(make_term_query(term)) } - FieldType::I64(_) => { + FieldType::I64(_) | FieldType::TieBreaker => { let val = parse_value_from_user_text::(value, field_entry.name())?; let term = Term::from_field_i64(field, val); Ok(make_term_query(term)) diff --git a/quickwit/quickwit-search/src/tests.rs b/quickwit/quickwit-search/src/tests.rs index 321f7056b14..d15c14a3cec 100644 --- a/quickwit/quickwit-search/src/tests.rs +++ b/quickwit/quickwit-search/src/tests.rs @@ -808,7 +808,7 @@ async fn test_sort_by_tie_breaker() { // Each call creates a separate split. The value supplied for `tie_breaker` is ignored. test_sandbox .add_documents(vec![ - json!({"body": "a", "tie_breaker": 0u64}), + json!({"body": "a", "tie_breaker": 0i64}), json!({"body": "b"}), json!({"body": "c"}), ]) @@ -844,20 +844,20 @@ async fn test_sort_by_tie_breaker() { .map(|hit| { let partial_hit = hit.partial_hit.unwrap(); let Some(SortByValue { - sort_value: Some(SortValue::U64(tie_breaker)), + sort_value: Some(SortValue::I64(tie_breaker)), }) = partial_hit.sort_value else { - panic!("expected a u64 tie_breaker sort value"); + panic!("expected an i64 tie_breaker sort value"); }; (partial_hit.split_id, partial_hit.doc_id, tie_breaker) }) - .collect::>() + .collect::>() } }; let ascending_hits = search_hits(SortOrder::Asc).await; assert!(ascending_hits.is_sorted_by_key(|(_, _, tie_breaker)| *tie_breaker)); - let mut tie_breakers_per_split: BTreeMap> = BTreeMap::new(); + let mut tie_breakers_per_split: BTreeMap> = BTreeMap::new(); for (split_id, doc_id, tie_breaker) in &ascending_hits { tie_breakers_per_split .entry(split_id.clone()) @@ -870,7 +870,7 @@ async fn test_sort_by_tie_breaker() { tie_breakers.sort(); for (doc_offset, (doc_id, tie_breaker)) in tie_breakers.iter().enumerate() { assert_eq!(*doc_id as usize, doc_offset); - assert_eq!(*tie_breaker, tie_breakers[0].1 + doc_offset as u64); + assert_eq!(*tie_breaker, tie_breakers[0].1 + doc_offset as i64); } } @@ -902,6 +902,86 @@ async fn test_sort_by_tie_breaker() { test_sandbox.assert_quit().await; } +#[tokio::test] +async fn test_search_after_with_tie_breaker() { + let index_id = "search_after_with_tie_breaker"; + let doc_mapping_yaml = r#" + field_mappings: + - name: timestamp + type: datetime + fast: true + - name: tie_breaker + type: tie_breaker + "#; + let test_sandbox = TestSandbox::create(index_id, doc_mapping_yaml, "{}", &[]) + .await + .unwrap(); + // All documents share the primary sort value. Each call creates a separate split. + for num_docs in [3, 2] { + test_sandbox + .add_documents(vec![json!({"timestamp": "2025-04-01T00:00:00Z"}); num_docs]) + .await + .unwrap(); + } + + for order in [SortOrder::Asc, SortOrder::Desc] { + let mut request = SearchRequest { + index_id_patterns: vec![index_id.to_string()], + query_ast: serde_json::to_string(&QueryAst::MatchAll).unwrap(), + max_hits: 10, + sort_fields: ["timestamp", "tie_breaker"] + .into_iter() + .map(|field_name| SortField { + field_name: field_name.to_string(), + sort_order: order as i32, + sort_datetime_format: None, + }) + .collect(), + ..Default::default() + }; + let response = single_node_search( + request.clone(), + test_sandbox.metastore(), + test_sandbox.storage_resolver(), + ) + .await + .unwrap(); + assert_eq!(response.num_hits, 5); + assert_eq!(response.hits.len(), 5); + + request.max_hits = 1; + for hit in response.hits { + let expected_hit = hit.partial_hit.unwrap(); + assert!(matches!( + expected_hit.sort_value2, + Some(SortByValue { + sort_value: Some(SortValue::I64(_)), + }) + )); + let page = single_node_search( + request.clone(), + test_sandbox.metastore(), + test_sandbox.storage_resolver(), + ) + .await + .unwrap(); + assert_eq!(page.hits.len(), 1); + assert_eq!(page.hits[0].partial_hit.as_ref(), Some(&expected_hit)); + request.search_after = Some(expected_hit); + } + let page = single_node_search( + request, + test_sandbox.metastore(), + test_sandbox.storage_resolver(), + ) + .await + .unwrap(); + assert!(page.hits.is_empty()); + } + + test_sandbox.assert_quit().await; +} + #[tokio::test] async fn test_sort_by_2_field() { let index_id = "sort_by_dynamic_field".to_string();