From a054fdcb2d5e4487099eaadfd5cec879841d9bd5 Mon Sep 17 00:00:00 2001
From: MWR27 <64335495+MWR27@users.noreply.github.com>
Date: Tue, 14 Jul 2026 12:34:59 -0400
Subject: [PATCH] Add foundation for automatic refactoring tool

---
 pygrate2/__init__.py            |   0
 pygrate2/__main__.py            |   3 +
 pygrate2/fixes/__init__.py      |   0
 pygrate2/fixes/fix_sem_div.py   |  30 ++++
 pygrate2/interactiverefactor.py | 284 ++++++++++++++++++++++++++++++++
 pygrate2/main.py                |  97 +++++++++++
 pygrate2/pygrate_helpers.py     |   2 +
 pygrate2/refactor_util.py       |  10 ++
 pygrate2/semfixer_base.py       |   9 +
 pygrate2/warninginfo.py         |   9 +
 10 files changed, 444 insertions(+)
 create mode 100644 pygrate2/__init__.py
 create mode 100644 pygrate2/__main__.py
 create mode 100644 pygrate2/fixes/__init__.py
 create mode 100644 pygrate2/fixes/fix_sem_div.py
 create mode 100644 pygrate2/interactiverefactor.py
 create mode 100644 pygrate2/main.py
 create mode 100644 pygrate2/pygrate_helpers.py
 create mode 100644 pygrate2/refactor_util.py
 create mode 100644 pygrate2/semfixer_base.py
 create mode 100644 pygrate2/warninginfo.py

diff --git a/pygrate2/__init__.py b/pygrate2/__init__.py
new file mode 100644
index 000000000000000..e69de29bb2d1d64
diff --git a/pygrate2/__main__.py b/pygrate2/__main__.py
new file mode 100644
index 000000000000000..7186d05800419da
--- /dev/null
+++ b/pygrate2/__main__.py
@@ -0,0 +1,3 @@
+from .main import main
+
+main()
\ No newline at end of file
diff --git a/pygrate2/fixes/__init__.py b/pygrate2/fixes/__init__.py
new file mode 100644
index 000000000000000..e69de29bb2d1d64
diff --git a/pygrate2/fixes/fix_sem_div.py b/pygrate2/fixes/fix_sem_div.py
new file mode 100644
index 000000000000000..ee129efd22b197d
--- /dev/null
+++ b/pygrate2/fixes/fix_sem_div.py
@@ -0,0 +1,30 @@
+from lib2to3.fixer_util import Name, Comma, Call
+from lib2to3.pygram import python_symbols
+from lib2to3.pgen2 import token
+from lib2to3.pytree import Node, Leaf
+
+from .. import semfixer_base
+
+class FixSemDiv(semfixer_base.SemanticFix):
+    HELPER_FUNCTION = u'classic_div'
+
+    def match(self, node):
+        if node.type == python_symbols.term:
+            for child in node.children[1::2]:
+                if child.type == token.SLASH:
+                    return True
+        return False
+
+    def transform(self, node, results):
+        dividend_nodes = []
+        term_children = iter(node.children)
+        for child in term_children:
+            if child.type == token.SLASH:
+                dividend = Node(python_symbols.term, dividend_nodes, prefix=u'')
+                divisor = term_children.next().clone()
+                divisor.prefix = u' '
+                dividend_nodes = [Call(Name(self.HELPER_FUNCTION), [dividend, Comma(), divisor])]
+                self.wrote_helper = True
+            else:
+                dividend_nodes.append(child.clone())
+        return Node(python_symbols.term, dividend_nodes, prefix=node.children[0].prefix)
diff --git a/pygrate2/interactiverefactor.py b/pygrate2/interactiverefactor.py
new file mode 100644
index 000000000000000..fffb7aedd222c30
--- /dev/null
+++ b/pygrate2/interactiverefactor.py
@@ -0,0 +1,284 @@
+import os
+import operator
+from itertools import chain
+import inspect
+
+from lib2to3.main import StdoutRefactoringTool
+from .semfixer_base import SemanticFix
+
+class InteractiveRefactoringTool(StdoutRefactoringTool):
+    def __init__(self, warning_fixers, options, explicit, nobackups, show_diffs,
+                 input_base_dir='', output_dir='', append_suffix=''):
+        self._warning_fixers = dict(warning_fixers)
+        self._warnings = None
+        self._skipped = []
+        self._helper_functions = set()
+        self._dir_name = None
+        self._handled_warnings = set()
+        
+        super(InteractiveRefactoringTool, self).__init__(self._warning_fixers.values(), options, explicit, nobackups, show_diffs,
+                                                        input_base_dir, output_dir, append_suffix)
+        self.write_unchanged_files = True
+
+    def get_fixers(self):
+        """Inspects the options to load the requested patterns and handlers.
+
+        Returns:
+          (pre_order, post_order), where pre_order is the list of fixers that
+          want a pre-order AST traversal, and post_order is the list that want
+          post-order traversal.
+        """
+        pre_order_fixers = []
+        post_order_fixers = []
+        for warning_msg, fix_mod_path in self._warning_fixers.iteritems():
+            mod = __import__(fix_mod_path, {}, {}, ["*"])
+            fix_name = fix_mod_path.rsplit(".", 1)[-1]
+            if fix_name.startswith(self.FILE_PREFIX):
+                fix_name = fix_name[len(self.FILE_PREFIX):]
+            parts = fix_name.split("_")
+            class_name = self.CLASS_PREFIX + "".join([p.title() for p in parts])
+            try:
+                fix_class = getattr(mod, class_name)
+            except AttributeError:
+                raise FixerError("Can't find %s.%s" % (fix_name, class_name))
+            if issubclass(fix_class, SemanticFix):
+                fixer = fix_class(self.options, self.fixer_log, None)
+            else:
+                fixer = fix_class(self.options, self.fixer_log)
+            if fixer.explicit and self.explicit is not True and \
+                    fix_mod_path not in self.explicit:
+                self.log_message("Skipping optional fixer: %s", fix_name)
+                continue
+
+            self.log_debug("Adding transformation: %s", fix_name)
+            if fixer.order == "pre":
+                pre_order_fixers.append(fixer)
+            elif fixer.order == "post":
+                post_order_fixers.append(fixer)
+            else:
+                raise FixerError("Illegal fixer order: %r" % fixer.order)
+            self._warning_fixers[warning_msg] = fixer
+
+        key_func = operator.attrgetter("run_order")
+        pre_order_fixers.sort(key=key_func)
+        post_order_fixers.sort(key=key_func)
+        return (pre_order_fixers, post_order_fixers)
+
+    def refactor_dir(self, dir_name, write=False, doctests_only=False, warnings=None):
+        self._warnings = warnings
+        self._dir_name = dir_name
+        super(InteractiveRefactoringTool, self).refactor_dir(dir_name, write=write, doctests_only=doctests_only)
+        if self._helper_functions:
+            with open(os.path.join(self._output_dir, 'pygrate_helpers.py'), 'w') as f:
+                f.write(create_helpers(self._helper_functions))
+        if self._skipped:
+            print 'Skipped warnings:'
+            for warning in self._skipped:
+                print warning
+        self._warnings = None
+        self._skipped = []
+        self._helper_functions = set()
+        self._dir_name = None
+    
+    def refactor_file(self, filename, write=False, doctests_only=False, warnings=None):
+        if warnings:
+            self._warnings = warnings
+        super(InteractiveRefactoringTool, self).refactor_file(filename, write=write, doctests_only=doctests_only)
+        if self._dir_name is None:
+            self._skipped = []
+
+    def refactor_tree(self, tree, name):
+        """Refactors a parse tree (modifying the tree in place).
+
+        For compatible patterns the bottom matcher module is
+        used. Otherwise the tree is traversed node-to-node for
+        matches.
+
+        Args:
+            tree: a pytree.Node instance representing the root of the tree
+                  to be refactored.
+            name: a human-readable name for this tree.
+
+        Returns:
+            True if the tree was modified, False otherwise.
+        """
+        helper_functions = set()
+
+        for fixer in chain(self.pre_order, self.post_order):
+            fixer.start_tree(tree, name)
+
+        #use traditional matching for the incompatible fixers
+        pre_pair = self._traverse_by(self.bmi_pre_order_heads, tree.pre_order())
+        post_pair = self._traverse_by(self.bmi_post_order_heads, tree.post_order())
+
+        helper_functions |= pre_pair[0]
+        helper_functions |= post_pair[0]
+
+        self._skipped += pre_pair[1]
+        self._skipped += post_pair[1]
+
+        # obtain a set of candidate nodes
+        match_set = self.BM.run(tree.leaves())
+
+        while any(match_set.values()):
+            for fixer in self.BM.fixers:
+                if fixer in match_set and match_set[fixer]:
+                    #sort by depth; apply fixers from bottom(of the AST) to top
+                    match_set[fixer].sort(key=pytree.Base.depth, reverse=True)
+
+                    if fixer.keep_line_order:
+                        #some fixers(eg fix_imports) must be applied
+                        #with the original file's line order
+                        match_set[fixer].sort(key=pytree.Base.get_lineno)
+
+                    for node in list(match_set[fixer]):
+                        if node in match_set[fixer]:
+                            match_set[fixer].remove(node)
+
+                        try:
+                            find_root(node)
+                        except ValueError:
+                            # this node has been cut off from a
+                            # previous transformation ; skip
+                            continue
+
+                        if node.fixers_applied and fixer in node.fixers_applied:
+                            # do not apply the same fixer again
+                            continue
+
+                        results = fixer.match(node)
+
+                        if results:
+                            new = fixer.transform(node, results)
+                            if new is not None:
+                                node.replace(new)
+                                #new.fixers_applied.append(fixer)
+                                for node in new.post_order():
+                                    # do not apply the fixer again to
+                                    # this or any subnode
+                                    if not node.fixers_applied:
+                                        node.fixers_applied = []
+                                    node.fixers_applied.append(fixer)
+
+                                # update the original match set for
+                                # the added code
+                                new_matches = self.BM.run(new.leaves())
+                                for fxr in new_matches:
+                                    if not fxr in match_set:
+                                        match_set[fxr]=[]
+
+                                    match_set[fxr].extend(new_matches[fxr])
+
+        for fixer in chain(self.pre_order, self.post_order):
+            fixer.finish_tree(tree, name)
+
+        if self._dir_name is None:
+            if helper_functions:
+                self.insert_helper_defs(tree, helper_functions)
+            if self._skipped:
+                print 'Skipped warnings:'
+                for warning in self._skipped:
+                    print warning
+            self._warnings = None
+            self._skipped = []
+            self._helper_functions = set()
+        elif helper_functions:
+            self.insert_helper_import(tree, helper_functions)
+            self._helper_functions |= helper_functions
+        return tree.was_changed
+
+    def _traverse_by(self, fixers, traversal):
+        """Traverse an AST, applying a set of fixers to each node.
+
+        This is a helper method for refactor_tree().
+
+        Args:
+            fixers: a list of fixer instances.
+            traversal: a generator that yields AST nodes.
+
+        Returns:
+            Pair of helper functions used and skipped warnings
+        """
+        helper_functions = set()
+        skipped = []
+        if not fixers:
+            return (helper_functions, skipped)
+        for node in traversal:
+            for warning in self._warnings:
+                if warning in self._handled_warnings or node.get_lineno() != warning.lineno:
+                    continue
+                try:
+                    fixer = self._warning_fixers[warning.msg]
+                    if fixer not in fixers[node.type]:
+                        continue
+                except:
+                    print 'No fixer for warning: {}\n'.format(warning)
+                    self._handled_warnings.add(warning)
+                    continue
+                results = fixer.match(node)
+                if results:
+                    potential = node.clone()
+                    new = fixer.transform(potential, results)
+                    # for when the fixer replaces and returns nothing
+                    if new is None:
+                        new = potential
+                    print '{}:{}: {}'.format(warning.filename, warning.lineno, warning.msg)
+                    print 'Could safely refactor:'
+
+                    try:
+                        if self.output_lock is not None:
+                            with self.output_lock:
+                                print_diff(unicode(node), unicode(new))
+                                sys.stdout.flush()
+                        else:
+                            print_diff(unicode(node), unicode(new))
+                    except UnicodeEncodeError:
+                        warn("couldn't encode %s's diff for your terminal" %
+                            (name,))
+                        return
+
+                    print 'Apply refactoring? [y/n]'
+                    while True:
+                        answer = raw_input('> ')
+                        if answer == 'y':
+                            node.replace(new)
+                            node = new
+                            if fixer.wrote_helper:
+                                helper_functions.add(fixer.HELPER_FUNCTION)
+                        elif answer == 'n':
+                            skipped.append(warning)
+                        else:
+                            print 'Unknown command. Apply refactoring? [y/n]'
+                            continue
+                        break
+                    self._handled_warnings.add(warning)
+        return (helper_functions, skipped)
+    
+    def insert_helper_defs(self, tree, helper_functions):
+        helper_tree = self.driver.parse_string(create_helpers(helper_functions))
+        helper_tree.children[0].prefix = u'\n'
+        for child in helper_tree.children[:-1]:
+            tree.insert_child(-1, child)
+        
+
+    def insert_helper_import(self, tree, helper_functions):
+        import_string = 'from {}.pygrate_helpers import '.format(os.path.basename(self._dir_name))
+        helper_iter = iter(helper_functions)
+        import_string += helper_iter.next()
+        for helper in helper_iter:
+            import_string += ', {}'.format(helper)
+        import_stmt_tree = self.driver.parse_string('{}\n'.format(import_string))
+        tree.insert_child(0, import_stmt_tree.children[0])
+
+def create_helpers(helper_functions):
+    string = u''
+    mod = __import__('pygrate2.pygrate_helpers', {}, {}, list(helper_functions))
+    for helper in helper_functions:
+        string += inspect.getsource(getattr(mod, helper)) + '\n'
+    return string
+
+def print_diff(old, new):
+    for line in old.splitlines():
+        print '- {}'.format(line)
+    for line in new.splitlines():
+        print '+ {}'.format(line)
diff --git a/pygrate2/main.py b/pygrate2/main.py
new file mode 100644
index 000000000000000..27715d398891885
--- /dev/null
+++ b/pygrate2/main.py
@@ -0,0 +1,97 @@
+import sys
+import os
+import subprocess
+import re
+import optparse
+
+from lib2to3 import refactor
+from .interactiverefactor import InteractiveRefactoringTool
+from .warninginfo import WarningInfo
+
+def main():
+    parser = optparse.OptionParser(usage="pygrate2 [options] source dest")
+    parser.add_option("-a", "--argv",
+                        help="File that contains arguments for the program. The program is run for each line.")
+
+    options, args = parser.parse_args()
+
+    if len(args) == 0:
+        parser.print_help()
+        return
+    if len(args) == 1:
+        print 'Missing dest argument'
+        return
+    if len(args) > 2:
+        print 'Too many arguments'
+        return
+    if not os.path.exists(args[0]):
+        print 'Source path does not exist'
+        return
+    if options.argv and os.path.exists(options.argv):
+        print 'argv path does not exist'
+        return
+    
+    source_path = os.path.abspath(args[0])
+    dest_path = os.path.abspath(args[1])
+
+    input_base_dir = source_path
+    if (not input_base_dir.endswith(os.sep) and not os.path.isdir(input_base_dir)):
+        input_base_dir = os.path.dirname(input_base_dir)
+    input_base_dir = input_base_dir.rstrip(os.sep)
+
+    warnings = set()
+    if not options.argv:
+        warnings = run_program(source_path)
+    else:
+        with open(options.argv, 'r') as f:
+            for line in file:
+                prog_warnings = run_program(source_path, line.split())
+                warnings |= prog_warnings
+
+    warning_fixers = {'classic int division': 'fix_sem_div'}
+    generate_fixer_paths(warning_fixers)
+
+    tool = InteractiveRefactoringTool(
+            warning_fixers=warning_fixers,
+            options=None,
+            explicit=None,
+            nobackups=True,
+            show_diffs=True,
+            input_base_dir=input_base_dir,
+            output_dir=dest_path)
+
+    if os.path.isdir(source_path):
+        tool.refactor_dir(source_path, warnings=warnings, write=True)
+    else:
+        tool.refactor_file(source_path, warnings=warnings, write=True)
+
+def run_program(source, args=None):
+    proc_args = [sys.executable, '-3']
+    env = os.environ.copy()
+    source_arg = source
+    if os.path.isdir(source):
+        proc_args.append('-m')
+        env['PYTHONPATH'] = os.path.dirname(source.rstrip(os.sep))
+        source_arg = os.path.basename(source.rstrip(os.sep))
+    proc_args.append(source_arg)
+    if args:
+        proc_args += args
+
+    proc = subprocess.Popen(proc_args, stderr=subprocess.PIPE, env=env)
+    stdoutdata, stderrdata = proc.communicate()
+
+    warning_pattern = re.compile(r"(.+):(\d+): (.*)Warning: (.*)")
+
+    warnings = set()
+    for line in stderrdata.splitlines():
+        m = re.match(warning_pattern, line)
+
+        if m is not None:
+            warning = WarningInfo(m.group(1), int(m.group(2)), m.group(3), m.group(4))
+            warnings.add(warning)
+
+    return warnings
+
+def generate_fixer_paths(warning_fixers):
+    for warning, fixer_name in warning_fixers.iteritems():
+        warning_fixers[warning] = 'pygrate2.fixes.' + fixer_name
diff --git a/pygrate2/pygrate_helpers.py b/pygrate2/pygrate_helpers.py
new file mode 100644
index 000000000000000..43a1be9d5f8ec95
--- /dev/null
+++ b/pygrate2/pygrate_helpers.py
@@ -0,0 +1,2 @@
+def classic_div(dividend, divisor):
+    return dividend // divisor if isinstance(dividend, int) and isinstance(divisor, int) else dividend / divisor
diff --git a/pygrate2/refactor_util.py b/pygrate2/refactor_util.py
new file mode 100644
index 000000000000000..13c96672077bdc0
--- /dev/null
+++ b/pygrate2/refactor_util.py
@@ -0,0 +1,10 @@
+import re
+import ast
+
+from lib2to3.pytree import Leaf
+
+# assumes FixNumliterals was run
+def is_integer_literal(node):
+    if isinstance(node, Leaf):
+        return re.match('(?:[1-9][0-9]*|0(?:[oO][0-7]+|[xX][0-9a-fA-F]+|[bB][01]+)?)$', node.value) is not None
+    return False
diff --git a/pygrate2/semfixer_base.py b/pygrate2/semfixer_base.py
new file mode 100644
index 000000000000000..cbd132ba011d9a8
--- /dev/null
+++ b/pygrate2/semfixer_base.py
@@ -0,0 +1,9 @@
+from lib2to3.fixer_base import BaseFix
+
+class SemanticFix(BaseFix):
+    HELPER_FUNCTION = None
+
+    def __init__(self, options, log, program_context=None):
+        super(SemanticFix, self).__init__(options, log)
+        self._program_context = program_context
+        self.wrote_helper = False # whether the last call to transform refactored using HELPER_FUNCTION
diff --git a/pygrate2/warninginfo.py b/pygrate2/warninginfo.py
new file mode 100644
index 000000000000000..97a54a41ba7711c
--- /dev/null
+++ b/pygrate2/warninginfo.py
@@ -0,0 +1,9 @@
+class WarningInfo(object):
+    def __init__(self, filename, lineno, warning_type, msg):
+        self.filename = filename
+        self.lineno = lineno
+        self.warning_type = warning_type
+        self.msg = msg
+
+    def __str__(self):
+        return '{}:{}: {}Warning: {}'.format(self.filename, self.lineno, self.warning_type, self.msg)