gh-159672: Cover keyword repairs across logical statements

Author: pablogsal

Lib/test/test_syntax.py

@@ -287,11 +287,11 @@
 
 >>> [x for x if range(1)]
 Traceback (most recent call last):
-SyntaxError: 'in' expected after for-loop variables
+SyntaxError: invalid syntax. Did you mean 'in'?
 
 >>> tuple(x for x if range(1))
 Traceback (most recent call last):
-SyntaxError: 'in' expected after for-loop variables
+SyntaxError: invalid syntax. Did you mean 'in'?
 
 >>> [x for x() in a]
 Traceback (most recent call last):

Lib/test/test_traceback.py

@@ -1815,6 +1815,10 @@ class TestKeywordTypoSuggestions(unittest.TestCase):
         ("[x for x\nin range(3)\nof x]", "if"),
         ("[123 fur x\nin range(3)\nif x]", "for"),
         ("for x im n:\n  pass", "in"),
+        ("for x is n:\n  pass", "in"),
+        ("is x in y:\n  pass", "if"),
+        ("try:\n  pass\nfinaly:\n  pass", "finally"),
+        ("result = (one\n          adn two)", "and"),
         ("mach x:", "match"),
         ("math x:", "match"),
         ("match 1:\n  cse 1:", "case"),
@@ -1873,17 +1877,22 @@ def test_no_keyword_suggestion_for_comma_errors(self):
     file=sys.stderr
 )
 '''
-        source = textwrap.dedent(code).strip()
-        rc, stdout, stderr = assert_python_failure('-c', source)
-        stderr_text = stderr.decode('utf-8')
-        self.assertIn("Perhaps you forgot a comma", stderr_text)
-        self.assertNotIn("Did you mean", stderr_text)
+        for source in (textwrap.dedent(code).strip(), "[foo bar]", "f(foo bar)"):
+            with self.subTest(source=source):
+                rc, stdout, stderr = assert_python_failure('-c', source)
+                stderr_text = stderr.decode('utf-8')
+                self.assertIn("Perhaps you forgot a comma", stderr_text)
+                self.assertNotIn("Did you mean", stderr_text)
 
     @cpython_only
     def test_parser_suggestions_need_no_formatting_work(self):
         cases = [
             ("def f():\n    returm value", "return", 2, 5),
+            ("a=b=c=d=e=f=g=h=i=0\ndef fn():\n    retrun True", "return", 3, 5),
+            ("def f(a,b):\n    if b:\n        yield\n    retrun a+b", "return", 4, 5),
+            ("def f():\n    for i in range(10):\n        yield i\n\ndfe x():", "def", 5, 1),
             ("é = 1; impor math", "import", 1, 8),
+            ("for café is items:\n    pass", "in", 1, 10),
             ("a = 1\nif True:\n    pass\nelso:\n    pass", "else", 4, 1),
             ("[123 fur x\nin range(3)\nif x]", "for", 1, 6),
             ("mach x:\n    case 1: pass", "match", 1, 1),

Parser/action_helpers.c

@@ -986,6 +986,7 @@ _PyPegen_raise_error_for_missing_comma(Parser *p, expr_ty a, expr_ty b)
     if (p->tokens[p->mark - 1]->level == 0) {
         return NULL;
     }
+    p->missing_comma = 1;
     // For multi-line expressions (like string concatenations), point to the
     // last line instead of the first for a more helpful error message.
     // Use a->col_offset as the starting column since all strings in the

Parser/pegen.c

@@ -1022,6 +1022,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
 #endif
     p->level = 0;
     p->call_invalid_rules = 0;
+    p->missing_comma = 0;
     p->input_exhausted = 0;
     p->keyword_trial = NULL;
 #ifdef Py_DEBUG
@@ -1107,6 +1108,22 @@ add_keyword_typo(KeywordTypo *candidates, int *count, int mark,
         max_cost = Py_MIN(max_cost, 2);
     }
     Py_ssize_t cost = _Py_UTF8_Edit_Cost(name, word, max_cost);
+    // Adjacent transpositions are one spelling error. The shared Levenshtein
+    // distance counts them as two substitutions.
+    if (cost > 2 && PyUnicode_IS_ASCII(name) &&
+        PyUnicode_GET_LENGTH(name) == PyUnicode_GET_LENGTH(word))
+    {
+        const char *text = (const char *)PyUnicode_1BYTE_DATA(name);
+        Py_ssize_t i = 0;
+        while (text[i] && text[i] == keyword[i]) {
+            i++;
+        }
+        if (text[i] && text[i + 1] && text[i] == keyword[i + 1] &&
+            text[i + 1] == keyword[i] && strcmp(text + i + 2, keyword + i + 2) == 0)
+        {
+            cost = 2;
+        }
+    }
     Py_DECREF(word);
     if (cost < 0) {
         return -1;
@@ -1227,23 +1244,28 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error)
         p->fill == 0 || p->fill > MAX_KEYWORD_TYPO_TOKENS ||
         _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES ||
         p->tokens[p->fill - 1]->type != ENDMARKER ||
-        _PyTokenizer_IsInteractive(p->tok) || !PyUnicode_Check(error->msg))
+        _PyTokenizer_IsInteractive(p->tok) ||
+        !Py_IS_TYPE(error, (PyTypeObject *)PyExc_SyntaxError))
     {
         return NULL;
     }
-    const char *message = PyUnicode_AsUTF8(error->msg);
-    if (message == NULL || (strcmp(message, "invalid syntax") != 0 &&
-                           strstr(message, "Perhaps you forgot a comma") == NULL))
-    {
-        return NULL;
-    }
-    // Prefer an existing missing-comma diagnostic unless the spelling differs
-    // by only one edit (the shared edit-cost function charges two per edit).
-    Py_ssize_t max_cost = strcmp(message, "invalid syntax") == 0 ? 4 : 2;
     long error_line = PyLong_AsLong(error->lineno);
-    if (error_line < 0) {
+    long error_offset = PyLong_AsLong(error->offset);
+    if (error_line < 0 || error_offset < 0) {
         return NULL;
     }
+    Py_ssize_t error_col = -1;
+    if (error_offset > 0 && error->text != NULL && PyUnicode_Check(error->text)) {
+        PyObject *prefix = PyUnicode_Substring(error->text, 0, error_offset - 1);
+        if (prefix == NULL) {
+            return NULL;
+        }
+        const char *text = PyUnicode_AsUTF8AndSize(prefix, &error_col);
+        Py_DECREF(prefix);
+        if (text == NULL) {
+            return NULL;
+        }
+    }
     KeywordTypo candidates[MAX_KEYWORD_TYPO_CANDIDATES];
     int count = 0;
     int start = 0;
@@ -1261,15 +1283,44 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error)
     };
     for (int i = start; i < p->fill; i++) {
         Token *token = p->tokens[i];
-        if (token->type != NAME || token->lineno > error_line ||
-            PyBytes_GET_SIZE(token->bytes) < 2 ||
-            PyBytes_GET_SIZE(token->bytes) > 10)
-        {
+        // Error spans can start before the typo in a multiline expression.
+        // Keep the complete logical statement as the candidate range.
+        if (token->type == NEWLINE) {
+            break;
+        }
+        const char *text = token->type == NAME ? PyBytes_AS_STRING(token->bytes) : NULL;
+        Py_ssize_t text_length = text == NULL ? 0 : PyBytes_GET_SIZE(token->bytes);
+        int is_keyword = token->type != NAME;
+        // A keyword can itself be a typo, as in "for x is y". Consider an
+        // existing keyword only at the reported error, not throughout the line.
+        if (text == NULL && token->lineno == error_line && token->col_offset == error_col) {
+            for (int length = 0; length < p->n_keyword_lists && text == NULL; length++) {
+                for (KeywordToken *k = p->keywords[length];
+                     k != NULL && k->type != -1; k++)
+                {
+                    if (k->type == token->type) {
+                        text = k->str;
+                        text_length = length;
+                        break;
+                    }
+                }
+            }
+        }
+        if (text == NULL || text_length < 2 || text_length > 10) {
+            continue;
+        }
+        for (char **k = p->soft_keywords; k != NULL && *k != NULL; k++) {
+            if (strcmp(text, *k) == 0) {
+                is_keyword = 1;
+                break;
+            }
+        }
+        if (is_keyword && (token->lineno != error_line || token->col_offset != error_col)) {
             continue;
         }
         const char *alias = NULL;
         for (size_t j = 0; j < Py_ARRAY_LENGTH(aliases); j++) {
-            if (strcmp(PyBytes_AS_STRING(token->bytes), aliases[j].name) == 0) {
+            if (strcmp(text, aliases[j].name) == 0) {
                 alias = aliases[j].keyword;
                 break;
             }
@@ -1281,20 +1332,29 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error)
             candidates[count++] = (KeywordTypo){i, alias, 0, 1};
             continue;
         }
-        PyObject *name = PyUnicode_DecodeUTF8(
-            PyBytes_AS_STRING(token->bytes), PyBytes_GET_SIZE(token->bytes), NULL);
+        PyObject *name = PyUnicode_DecodeUTF8(text, text_length, NULL);
         if (name == NULL) {
             return NULL;
         }
+        // Preserve missing-comma diagnostics and correctly spelled keywords
+        // unless one edit repairs the input. For keywords, only substitute a
+        // character: deleting one from "for" must not suggest "or".
+        Py_ssize_t max_cost = p->missing_comma || is_keyword ? 2 : 4;
         int result = 0;
         for (int length = 0; length < p->n_keyword_lists && result == 0; length++) {
+            if (is_keyword && length != text_length) {
+                continue;
+            }
             for (KeywordToken *k = p->keywords[length];
                  k != NULL && k->type != -1 && result == 0; k++)
             {
                 result = add_keyword_typo(candidates, &count, i, name, k->str, max_cost);
             }
         }
         for (char **k = p->soft_keywords; k != NULL && *k != NULL && result == 0; k++) {
+            if (is_keyword && (Py_ssize_t)strlen(*k) != text_length) {
+                continue;
+            }
             result = add_keyword_typo(candidates, &count, i, name, *k, max_cost);
         }
         Py_DECREF(name);

Parser/pegen.h

@@ -103,6 +103,7 @@ typedef struct {
     Token *known_err_token;
     int level;
     int call_invalid_rules;
+    int missing_comma;
     KeywordTrial *keyword_trial;
     int debug;
     IdentifierCacheEntry *identifier_cache;