Fix: convert-workload produces an unparseable date bound, and loses a day on a whole-day lte

solrorbit/conversion/query.py

@@ -26,7 +26,7 @@
 """
 
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from .field import normalize_field_name
 
@@ -183,21 +183,25 @@ def _translate_query_node(node: dict, fq_list: list = None) -> str:
             field = normalize_field_name(field)
             # gt/lt are exclusive; Solr spells that with a curly bracket.
             if "gte" in bounds:
-                lo, lo_bracket = bounds["gte"], "["
+                lo, lo_bracket, lo_key = bounds["gte"], "[", "gte"
             elif "gt" in bounds:
-                lo, lo_bracket = bounds["gt"], "{"
+                lo, lo_bracket, lo_key = bounds["gt"], "{", "gt"
             else:
-                lo, lo_bracket = "*", "["
+                lo, lo_bracket, lo_key = "*", "[", None
             if "lte" in bounds:
-                hi, hi_bracket = bounds["lte"], "]"
+                hi, hi_bracket, hi_key = bounds["lte"], "]", "lte"
             elif "lt" in bounds:
-                hi, hi_bracket = bounds["lt"], "}"
+                hi, hi_bracket, hi_key = bounds["lt"], "}", "lt"
             else:
-                hi, hi_bracket = "*", "]"
+                hi, hi_bracket, hi_key = "*", "]", None
             # Convert dates if format is specified (common for date fields)
             os_format = bounds.get("format")
-            lo = _convert_date_to_solr_format(lo, os_format)
-            hi = _convert_date_to_solr_format(hi, os_format)
+            lo, lo_is_date_only = _convert_date_to_solr_format(lo, os_format)
+            hi, hi_is_date_only = _convert_date_to_solr_format(hi, os_format)
+            if hi_is_date_only and hi_key == "lte":
+                hi, hi_bracket = _round_date_only_bound(hi), "}"
+            if lo_is_date_only and lo_key == "gt":
+                lo, lo_bracket = _round_date_only_bound(lo), "["
             return f"{field}:{lo_bracket}{lo} TO {hi}{hi_bracket}"
 
     if "exists" in node:
@@ -529,7 +533,18 @@ def _calendar_interval_to_solr_gap(interval: str) -> str:
     return mapping.get(str(interval).lower(), "+1MONTH")
 
 
-def _convert_date_to_solr_format(date_str, os_format=None) -> str:
+OS_TO_PYTHON_FORMAT = {
+    "dd/MM/yyyy": ("%d/%m/%Y", False),
+    "MM/dd/yyyy": ("%m/%d/%Y", False),
+    "yyyy-MM-dd": ("%Y-%m-%d", False),
+    "yyyy/MM/dd": ("%Y/%m/%d", False),
+    "dd-MM-yyyy": ("%d-%m-%Y", False),
+    "MM-dd-yyyy": ("%m-%d-%Y", False),
+    "yyyy-MM-dd HH:mm:ss": ("%Y-%m-%d %H:%M:%S", True),
+}
+
+
+def _convert_date_to_solr_format(date_str, os_format=None) -> tuple:
     """
     Convert an OpenSearch date string to Solr ISO 8601 format.
 
@@ -538,50 +553,60 @@ def _convert_date_to_solr_format(date_str, os_format=None) -> str:
         os_format: Optional OpenSearch date format pattern (e.g., "dd/MM/yyyy")
 
     Returns:
-        ISO 8601 date string for Solr (e.g., "2015-01-01T00:00:00Z")
+        (value, is_date_only) — the ISO 8601 date string for Solr
+        (e.g., "2015-01-01T00:00:00Z"), and whether the source named a whole
+        day rather than an instant.
 
     If the date is already in ISO format or conversion fails, returns the
     original string unchanged.
     """
     if not isinstance(date_str, str) or date_str in ("*", "now"):
-        return date_str
-
-    # Map OpenSearch date format patterns to Python strptime format
-    OS_TO_PYTHON_FORMAT = {
-        "dd/MM/yyyy": "%d/%m/%Y",
-        "MM/dd/yyyy": "%m/%d/%Y",
-        "yyyy-MM-dd": "%Y-%m-%d",
-        "yyyy/MM/dd": "%Y/%m/%d",
-        "dd-MM-yyyy": "%d-%m-%Y",
-        "MM-dd-yyyy": "%m-%d-%Y",
-        # Add more as needed
-    }
+        return date_str, False
 
     # If format is provided, use it to parse the date
     if os_format:
-        python_fmt = OS_TO_PYTHON_FORMAT.get(os_format)
-        if python_fmt:
+        pattern = OS_TO_PYTHON_FORMAT.get(os_format)
+        if pattern:
+            python_fmt, has_time = pattern
             try:
                 dt = datetime.strptime(date_str, python_fmt)
-                return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+                return dt.strftime("%Y-%m-%dT%H:%M:%SZ"), not has_time
             except ValueError:
                 logger.warning(f"Failed to parse date '{date_str}' with format '{os_format}'")
-                return date_str
+                return date_str, False
         else:
             logger.warning(f"Unknown OpenSearch date format: '{os_format}'")
 
     # Try common patterns if no format specified
-    for python_fmt in OS_TO_PYTHON_FORMAT.values():
+    for python_fmt, has_time in OS_TO_PYTHON_FORMAT.values():
         try:
             dt = datetime.strptime(date_str, python_fmt)
-            return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+            return dt.strftime("%Y-%m-%dT%H:%M:%SZ"), not has_time
         except ValueError:
             continue
 
     # If it's already in ISO-like format, return as-is
     # (handles cases like "2015-01-01T00:00:00Z" or partial ISO)
-    if "T" in date_str or len(date_str) == 10:  # YYYY-MM-DD
-        return date_str
+    if "T" in date_str:
+        return date_str, False
 
     logger.warning(f"Could not parse date '{date_str}', using as-is")
-    return date_str
+    return date_str, False
+
+
+def _round_date_only_bound(value: str) -> str:
+    """
+    Advance a whole-day bound to the start of the following day.
+
+    A date without a time names a day, and OpenSearch rounds it to the edge of
+    that day: `lte` and `gt` go to its LAST millisecond, `gte` and `lt` to its
+    first. Solr rounds nothing, so only the two that move to the end of the day
+    need translating, and naming the next day's first instant says that without
+    depending on how fine Solr's date precision happens to be.
+    """
+    try:
+        dt = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
+    except ValueError:
+        logger.warning(f"Could not round whole-day bound '{value}', using as-is")
+        return value
+    return (dt + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ")

tests/unit/solr/test_workload_converter.py

@@ -228,6 +228,58 @@ def test_a_missing_bound_is_open_and_inclusive(self):
         self.assertEqual("fare_amount:[* TO 100}", self._range({"lt": 100}))
         self.assertEqual("fare_amount:[* TO 100]", self._range({"lte": 100}))
 
+    def _date_range(self, bounds):
+        return translate_to_solr_json_dsl(
+            {"query": {"range": {"dropoff_datetime": bounds}}})["query"]
+
+    def test_a_datetime_with_a_space_separator_is_converted(self):
+        self.assertEqual(
+            "dropoff_datetime:[2015-01-01T00:00:00Z TO 2016-01-01T00:00:00Z}",
+            self._date_range({"gte": "2015-01-01 00:00:00", "lt": "2016-01-01 00:00:00"}))
+
+    def test_a_whole_day_lte_covers_that_day(self):
+        self.assertEqual(
+            "dropoff_datetime:[2015-01-01T00:00:00Z TO 2015-01-22T00:00:00Z}",
+            self._date_range({"gte": "01/01/2015", "lte": "21/01/2015",
+                              "format": "dd/MM/yyyy"}))
+
+    def test_a_whole_day_gt_excludes_that_day(self):
+        self.assertEqual(
+            "dropoff_datetime:[2015-01-02T00:00:00Z TO *]",
+            self._date_range({"gt": "01/01/2015", "format": "dd/MM/yyyy"}))
+
+    def test_the_bounds_that_round_down_do_not_move(self):
+        self.assertEqual(
+            "dropoff_datetime:[2015-01-01T00:00:00Z TO *]",
+            self._date_range({"gte": "01/01/2015", "format": "dd/MM/yyyy"}))
+        self.assertEqual(
+            "dropoff_datetime:[* TO 2015-01-01T00:00:00Z}",
+            self._date_range({"lt": "01/01/2015", "format": "dd/MM/yyyy"}))
+
+    def test_only_the_bound_that_is_used_is_rounded(self):
+        self.assertEqual(
+            "dropoff_datetime:[2015-01-01T00:00:00Z TO 2015-01-22T00:00:00Z}",
+            self._date_range({"gte": "01/01/2015", "gt": "05/01/2015",
+                              "lte": "21/01/2015", "lt": "10/01/2015",
+                              "format": "dd/MM/yyyy"}))
+
+    def test_a_bound_that_is_not_a_date_keeps_its_bracket(self):
+        for value in ("0000000010", "not-a-date", "2015-02-30"):
+            self.assertEqual(
+                f"serial_no:[0000000001 TO {value}]",
+                translate_to_solr_json_dsl(
+                    {"query": {"range": {"serial_no": {"gte": "0000000001",
+                                                      "lte": value}}}})["query"])
+
+    def test_an_instant_is_not_rounded(self):
+        self.assertEqual(
+            "dropoff_datetime:[2015-01-01T00:00:00Z TO 2016-01-01T00:00:00Z]",
+            self._date_range({"gte": "2015-01-01T00:00:00Z",
+                              "lte": "2016-01-01T00:00:00Z"}))
+        self.assertEqual(
+            "dropoff_datetime:[* TO 2016-01-01T00:00:00Z]",
+            self._date_range({"lte": "2016-01-01 00:00:00"}))
+
     def test_bool_with_filter_goes_to_fq(self):
         body = {
             "query": {