[3.14] gh-154719: Preserve trailing whitespace in t-string interpolation expressions (#154762) (#157392)

Author: lpyu001

Lib/test/test_annotationlib.py

@@ -1869,7 +1869,7 @@ def nested():
         self.assertEqual(type_repr(t'''{ 0
             & 1
             | 2
-        }'''), 't"""{ 0\n            & 1\n            | 2}"""')
+        }'''), 't"""{ 0\n            & 1\n            | 2\n        }"""')
         self.assertEqual(
             type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'"
         )

Lib/test/test_fstring.py

@@ -1668,6 +1668,14 @@ def __repr__(self):
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
 
+        result = f'''{(
+            1,  # Force lexer metadata reconstruction.
+            "\"#")=}'''
+        self.assertEqual(
+            result,
+            '(\n            1,  \n            "\\"#")=(1, \'"#\')',
+        )
+
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')
         self.assertEqual(f'{"""# this is not a comment

Lib/test/test_tstring.py

@@ -136,10 +136,82 @@ def test_debug_specifier(self):
         # Test white space in debug specifier
         t = t"Value: {value = }"
         self.assertTStringEqual(
-            t, ("Value: value = ", ""), [(value, "value", "r")]
+            t, ("Value: value = ", ""), [(value, "value ", "r")]
         )
         self.assertEqual(fstring(t), "Value: value = 42")
 
+        # Explicit line continuations after the debug marker are part of
+        # the debug text, not the interpolation expression.
+        for template, strings, interpolation, rendered in (
+            (
+                t"""Value: {value =\
+}""",
+                ("Value: value =\\\n", ""),
+                (value, "value ", "r"),
+                "Value: value =\\\n42",
+            ),
+            (
+                t"""Value: {value =\
+!r}""",
+                ("Value: value =\\\n", ""),
+                (value, "value ", "r"),
+                "Value: value =\\\n42",
+            ),
+            (
+                t"""Value: {value =\
+:04}""",
+                ("Value: value =\\\n", ""),
+                (value, "value ", None, "04"),
+                "Value: value =\\\n0042",
+            ),
+            (
+                t"""Value: {value =\
+\
+}""",
+                ("Value: value =\\\n\\\n", ""),
+                (value, "value ", "r"),
+                "Value: value =\\\n\\\n42",
+            ),
+        ):
+            with self.subTest(template=template):
+                self.assertTStringEqual(template, strings, [interpolation])
+                self.assertEqual(fstring(template), rendered)
+
+    def test_interpolation_expression_whitespace(self):
+        x = 42
+        for template, expected in (
+            (t"{x}", "x"),
+            (t"{x }", "x "),
+            (t"{ x}", " x"),
+            (t"{ x }", " x "),
+            (t"{  x  }", "  x  "),
+            (t"""{
+  x
+}""", "\n  x\n"),
+            (t"{ x !r}", " x "),
+            (t"{ x :.2f}", " x "),
+            (t"{ x = }", " x "),
+            (t"{ x = !r}", " x "),
+            (t"{ x = :.2f}", " x "),
+            (t"{x == 42 = }", "x == 42 "),
+        ):
+            with self.subTest(template=template):
+                self.assertEqual(
+                    template.interpolations[0].expression,
+                    expected,
+                )
+
+    def test_interpolation_expression_with_reconstructed_metadata(self):
+        regular = t'''{(
+            1,  # Force lexer metadata reconstruction.
+            "\"#")}'''
+        debug = t'''{(
+            1,  # Force lexer metadata reconstruction.
+            "\"#")=}'''
+        expected = '(\n            1,  \n            "\\"#")'
+        self.assertEqual(regular.interpolations[0].expression, expected)
+        self.assertEqual(debug.interpolations[0].expression, expected)
+
     def test_raw_tstrings(self):
         path = r"C:\Users"
         t = rt"{path}\Documents"

Lib/test/test_unparse.py

@@ -216,6 +216,15 @@ def test_tstrings(self):
         self.check_ast_roundtrip('t""')
         self.check_ast_roundtrip("t'{(lambda x: x)}'")
         self.check_ast_roundtrip("t'{t'{x}'}'")
+        self.check_ast_roundtrip(
+            r"""t'''{(
+                1,  # Force lexer metadata reconstruction.
+                "\"#")}'''"""
+        )
+        self.check_ast_roundtrip(
+            r'''t"""Value: {value =\
+}"""'''
+        )
 
     def test_tstring_with_nonsensical_str_field(self):
         # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise

Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-17-20-49.gh-issue-154719.eI88Gs.rst

@@ -0,0 +1,7 @@
+Trailing whitespace in a t-string interpolation expression is now preserved
+in :attr:`string.templatelib.Interpolation.expression`, up to the closing ``}``
+or the conversion (``!``), format (``:``), or debug (``=``) delimiter.
+Explicit line continuations following a debug ``=`` remain part of the debug
+text and are excluded from :attr:`~string.templatelib.Interpolation.expression`.
+Escaped quotes are now preserved while reconstructing t-string interpolation
+metadata and f-string debug text containing comments.

Parser/action_helpers.c

@@ -1510,21 +1510,38 @@ _get_interpolation_conversion(Parser *p, Token *debug, ResultTokenWithMetadata *
 }
 
 static PyObject *
-_strip_interpolation_expr(PyObject *exprstr)
+_strip_interpolation_debug_expr(PyObject *exprstr)
 {
     Py_ssize_t len = PyUnicode_GET_LENGTH(exprstr);
 
-    for (Py_ssize_t i = len - 1; i >= 0; i--) {
-        Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, i);
-        if (_PyUnicode_IsWhitespace(c) || c == '=') {
+    /* Discard whitespace and explicit line continuations after the debug "="
+       but preserve whitespace before it. */
+    while (len > 0) {
+        int has_newline = 0;
+        while (len > 0) {
+            Py_UCS4 c = PyUnicode_READ_CHAR(exprstr, len - 1);
+            if (!_PyUnicode_IsWhitespace(c)) {
+                break;
+            }
+            if (c == '\r' || c == '\n') {
+                has_newline = 1;
+            }
             len--;
         }
-        else {
+        if (!has_newline || len == 0 ||
+            PyUnicode_READ_CHAR(exprstr, len - 1) != '\\')
+        {
             break;
         }
+        len--;
+    }
+
+    /* Preserve unexpected metadata instead of dropping source text. */
+    if (len == 0 || PyUnicode_READ_CHAR(exprstr, len - 1) != '=') {
+        return Py_NewRef(exprstr);
     }
 
-    return PyUnicode_Substring(exprstr, 0, len);
+    return PyUnicode_Substring(exprstr, 0, len - 1);
 }
 
 expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, ResultTokenWithMetadata *conversion,
@@ -1555,7 +1572,9 @@ expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, Resu
     }
 
     assert(exprstr != NULL);
-    PyObject *final_exprstr = _strip_interpolation_expr(exprstr);
+    PyObject *final_exprstr = debug
+        ? _strip_interpolation_debug_expr(exprstr)
+        : Py_NewRef(exprstr);
     if (!final_exprstr || _PyArena_AddPyObject(arena, final_exprstr) < 0) {
         Py_XDECREF(final_exprstr);
         return NULL;

Parser/lexer/lexer.c

@@ -177,8 +177,17 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
         while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
             char ch = tok_mode->last_expr_buffer[i];
 
+            // Copy escaped characters without interpreting the escaped
+            // character as a quote or comment marker.
+            if (ch == '\\') {
+                result[j++] = ch;
+                i++;
+                if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
+                    result[j++] = tok_mode->last_expr_buffer[i];
+                }
+            }
             // Handle string quotes
-            if (ch == '"' || ch == '\'') {
+            else if (ch == '"' || ch == '\'') {
                 // See comment above to understand this part
                 if (!in_string) {
                     in_string = 1;