Use signed i64 generated tie breakers + bump Tantivy

docs/configuration/index-config.md

@@ -405,7 +405,7 @@ If, in addition, `attributes` is set as a default search field, then `color:red`
 
 #### `tie_breaker` type
 
-The `tie_breaker` type is a field whose values are generated by Quickwit when a split is finalized. It behaves like a `u64` field with `fast: true`, `stored: false` and `indexed: false`, and these options cannot be changed. Use it as a secondary sort key to reduce ties among documents with equal primary sort values.
+The `tie_breaker` type is a field whose values are generated by Quickwit when a split is finalized. It behaves like an `i64` field with `fast: true`, `stored: false` and `indexed: false`, and these options cannot be changed. Use it as a secondary sort key to reduce ties among documents with equal primary sort values.
 
 ```yaml
 name: tie_breaker

quickwit/Cargo.toml

@@ -378,7 +378,7 @@ quickwit-telemetry-exporters = { path = "quickwit-telemetry-exporters" }
 quickwit-transport = { path = "quickwit-transport" }
 
 # This is the tip of tantivy main.
-tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "4abc20f924e031a91f89c8cb44575e302e68fb38", default-features = false, features = [
+tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "2e87c2eb0c8577c96c82a777f345bbf236a985fe", default-features = false, features = [
   "jitexpr",
   "lz4-compression",
   "mmap",
@@ -387,7 +387,7 @@ tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "4abc20f924e
   "columnar-zstd-compression",
 ] }
 # Must stay on the same rev as `tantivy`.
-tantivy-common = { git = "https://github.com/quickwit-oss/tantivy/", rev = "4abc20f924e031a91f89c8cb44575e302e68fb38" }
+tantivy-common = { git = "https://github.com/quickwit-oss/tantivy/", rev = "2e87c2eb0c8577c96c82a777f345bbf236a985fe" }
 tantivy-fst = "0.5"
 
 # This is actually not used directly the goal is to fix the version

quickwit/quickwit-doc-mapper/src/doc_mapper/tantivy_val_to_json.rs

@@ -244,8 +244,8 @@ pub fn formatted_tantivy_value_to_json(
         LeafType::F64(numeric_options) => value_to_float(value, numeric_options),
         LeafType::U64(numeric_options) => value_to_u64(value, numeric_options),
         LeafType::TieBreaker => {
-            // Only `output_format` is read by `value_to_u64`. The other options describe what a
-            // tie-breaker field is: a fast-only `u64` whose values are never parsed from documents.
+            // Only `output_format` is read by `value_to_i64`. The other options describe what a
+            // tie-breaker field is: a fast-only `i64` whose values are never parsed from documents.
             let numeric_options = QuickwitNumericOptions {
                 description: None,
                 stored: false,
@@ -254,7 +254,7 @@ pub fn formatted_tantivy_value_to_json(
                 coerce: false,
                 output_format: NumericOutputFormat::Number,
             };
-            value_to_u64(value, &numeric_options)
+            value_to_i64(value, &numeric_options)
         }
         LeafType::I64(numeric_options) => value_to_i64(value, numeric_options),
         LeafType::Json(_) => {
@@ -411,6 +411,16 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_tantivy_value_to_json_value_tie_breaker() {
+        for value in [i32::MIN as i64, -1, 0, i32::MAX as i64] {
+            assert_eq!(
+                formatted_tantivy_value_to_json(TantivyValue::I64(value), &LeafType::TieBreaker),
+                Some(serde_json::json!(value))
+            );
+        }
+    }
+
     #[test]
     fn test_tantivy_value_to_json_value_u64() {
         let numeric_options_number = QuickwitNumericOptions::default();

quickwit/quickwit-query/src/query_ast/range_query.rs

@@ -143,7 +143,7 @@ impl BuildTantivyAst for RangeQuery {
                 )
                 .into()
             }
-            tantivy::schema::FieldType::U64(_) | tantivy::schema::FieldType::TieBreaker => {
+            tantivy::schema::FieldType::U64(_) => {
                 let (lower_bound, upper_bound) =
                     convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
                 FastFieldRangeQuery::new(
@@ -152,7 +152,7 @@ impl BuildTantivyAst for RangeQuery {
                 )
                 .into()
             }
-            tantivy::schema::FieldType::I64(_) => {
+            tantivy::schema::FieldType::I64(_) | tantivy::schema::FieldType::TieBreaker => {
                 let (lower_bound, upper_bound) =
                     convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
                 FastFieldRangeQuery::new(
@@ -363,7 +363,7 @@ mod tests {
         let schema = schema_builder.build();
         let range_query = RangeQuery {
             field: "tie_breaker".to_string(),
-            lower_bound: Bound::Included(JsonLiteral::String("1980".to_string())),
+            lower_bound: Bound::Included(JsonLiteral::String("-1980".to_string())),
             upper_bound: Bound::Included(JsonLiteral::String("1989".to_string())),
         };
         let tantivy_ast = range_query
@@ -374,7 +374,7 @@ mod tests {
         assert_eq!(
             format!("{leaf:?}"),
             "FastFieldRangeQuery { bounds: BoundsRange { lower_bound: Included(Term(field=0, \
-             type=U64, 1980)), upper_bound: Included(Term(field=0, type=U64, 1989)) } }"
+             type=I64, -1980)), upper_bound: Included(Term(field=0, type=I64, 1989)) } }"
         );
     }
 

quickwit/quickwit-query/src/query_ast/term_query.rs

@@ -220,7 +220,7 @@ mod tests {
     fn test_term_query_with_tie_breaker() {
         let term_query = TermQuery {
             field: "tie_breaker".to_string(),
-            value: "42".to_string(),
+            value: "-42".to_string(),
         };
         let mut schema_builder = Schema::builder();
         schema_builder.add_tie_breaker_field("tie_breaker");
@@ -231,7 +231,7 @@ mod tests {
         let leaf = tantivy_query_ast.as_leaf().unwrap();
         assert_eq!(
             &format!("{leaf:?}"),
-            "TermQuery(Term(field=0, type=U64, 42))"
+            "TermQuery(Term(field=0, type=I64, -42))"
         );
     }
 }

quickwit/quickwit-query/src/query_ast/utils.rs

@@ -121,12 +121,12 @@ fn compute_query_with_field(
 ) -> Result<TantivyQueryAst, InvalidQuery> {
     let field_type = field_entry.field_type();
     match field_type {
-        FieldType::U64(_) | FieldType::TieBreaker => {
+        FieldType::U64(_) => {
             let val = parse_value_from_user_text::<u64>(value, field_entry.name())?;
             let term = Term::from_field_u64(field, val);
             Ok(make_term_query(term))
         }
-        FieldType::I64(_) => {
+        FieldType::I64(_) | FieldType::TieBreaker => {
             let val = parse_value_from_user_text::<i64>(value, field_entry.name())?;
             let term = Term::from_field_i64(field, val);
             Ok(make_term_query(term))

quickwit/quickwit-search/src/tests.rs

@@ -808,7 +808,7 @@ async fn test_sort_by_tie_breaker() {
     // Each call creates a separate split. The value supplied for `tie_breaker` is ignored.
     test_sandbox
         .add_documents(vec![
-            json!({"body": "a", "tie_breaker": 0u64}),
+            json!({"body": "a", "tie_breaker": 0i64}),
             json!({"body": "b"}),
             json!({"body": "c"}),
         ])
@@ -844,20 +844,20 @@ async fn test_sort_by_tie_breaker() {
                 .map(|hit| {
                     let partial_hit = hit.partial_hit.unwrap();
                     let Some(SortByValue {
-                        sort_value: Some(SortValue::U64(tie_breaker)),
+                        sort_value: Some(SortValue::I64(tie_breaker)),
                     }) = partial_hit.sort_value
                     else {
-                        panic!("expected a u64 tie_breaker sort value");
+                        panic!("expected an i64 tie_breaker sort value");
                     };
                     (partial_hit.split_id, partial_hit.doc_id, tie_breaker)
                 })
-                .collect::<Vec<(String, u32, u64)>>()
+                .collect::<Vec<(String, u32, i64)>>()
         }
     };
 
     let ascending_hits = search_hits(SortOrder::Asc).await;
     assert!(ascending_hits.is_sorted_by_key(|(_, _, tie_breaker)| *tie_breaker));
-    let mut tie_breakers_per_split: BTreeMap<String, Vec<(u32, u64)>> = BTreeMap::new();
+    let mut tie_breakers_per_split: BTreeMap<String, Vec<(u32, i64)>> = BTreeMap::new();
     for (split_id, doc_id, tie_breaker) in &ascending_hits {
         tie_breakers_per_split
             .entry(split_id.clone())
@@ -870,7 +870,7 @@ async fn test_sort_by_tie_breaker() {
         tie_breakers.sort();
         for (doc_offset, (doc_id, tie_breaker)) in tie_breakers.iter().enumerate() {
             assert_eq!(*doc_id as usize, doc_offset);
-            assert_eq!(*tie_breaker, tie_breakers[0].1 + doc_offset as u64);
+            assert_eq!(*tie_breaker, tie_breakers[0].1 + doc_offset as i64);
         }
     }
 
@@ -902,6 +902,86 @@ async fn test_sort_by_tie_breaker() {
     test_sandbox.assert_quit().await;
 }
 
+#[tokio::test]
+async fn test_search_after_with_tie_breaker() {
+    let index_id = "search_after_with_tie_breaker";
+    let doc_mapping_yaml = r#"
+            field_mappings:
+              - name: timestamp
+                type: datetime
+                fast: true
+              - name: tie_breaker
+                type: tie_breaker
+            "#;
+    let test_sandbox = TestSandbox::create(index_id, doc_mapping_yaml, "{}", &[])
+        .await
+        .unwrap();
+    // All documents share the primary sort value. Each call creates a separate split.
+    for num_docs in [3, 2] {
+        test_sandbox
+            .add_documents(vec![json!({"timestamp": "2025-04-01T00:00:00Z"}); num_docs])
+            .await
+            .unwrap();
+    }
+
+    for order in [SortOrder::Asc, SortOrder::Desc] {
+        let mut request = SearchRequest {
+            index_id_patterns: vec![index_id.to_string()],
+            query_ast: serde_json::to_string(&QueryAst::MatchAll).unwrap(),
+            max_hits: 10,
+            sort_fields: ["timestamp", "tie_breaker"]
+                .into_iter()
+                .map(|field_name| SortField {
+                    field_name: field_name.to_string(),
+                    sort_order: order as i32,
+                    sort_datetime_format: None,
+                })
+                .collect(),
+            ..Default::default()
+        };
+        let response = single_node_search(
+            request.clone(),
+            test_sandbox.metastore(),
+            test_sandbox.storage_resolver(),
+        )
+        .await
+        .unwrap();
+        assert_eq!(response.num_hits, 5);
+        assert_eq!(response.hits.len(), 5);
+
+        request.max_hits = 1;
+        for hit in response.hits {
+            let expected_hit = hit.partial_hit.unwrap();
+            assert!(matches!(
+                expected_hit.sort_value2,
+                Some(SortByValue {
+                    sort_value: Some(SortValue::I64(_)),
+                })
+            ));
+            let page = single_node_search(
+                request.clone(),
+                test_sandbox.metastore(),
+                test_sandbox.storage_resolver(),
+            )
+            .await
+            .unwrap();
+            assert_eq!(page.hits.len(), 1);
+            assert_eq!(page.hits[0].partial_hit.as_ref(), Some(&expected_hit));
+            request.search_after = Some(expected_hit);
+        }
+        let page = single_node_search(
+            request,
+            test_sandbox.metastore(),
+            test_sandbox.storage_resolver(),
+        )
+        .await
+        .unwrap();
+        assert!(page.hits.is_empty());
+    }
+
+    test_sandbox.assert_quit().await;
+}
+
 #[tokio::test]
 async fn test_sort_by_2_field() {
     let index_id = "sort_by_dynamic_field".to_string();