From f9bbd320aa18594323c8cb3a8512181d5e62c0fe Mon Sep 17 00:00:00 2001 From: Sahil Rajput <90515841+SahilRajputdotexe@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:15:21 +0530 Subject: [PATCH] Add transforms to remove unused imports, variables, definitions and unreachable code Each is an option that is disabled by default. They are repeated until no more code is removed, as removing code can leave other code unused. --- CHANGELOG.md | 9 + docs/source/minification_options/index.rst | 4 + .../remove_unreachable.py | 24 + .../remove_unreachable.rst | 38 ++ .../remove_unused_definitions.py | 25 + .../remove_unused_definitions.rst | 47 ++ .../remove_unused_imports.py | 12 + .../remove_unused_imports.rst | 46 ++ .../remove_unused_variables.py | 18 + .../remove_unused_variables.rst | 48 ++ src/python_minifier/__init__.py | 20 +- src/python_minifier/__init__.pyi | 6 +- src/python_minifier/__main__.py | 28 + src/python_minifier/transforms/dead_code.py | 54 ++ .../transforms/remove_unreachable.py | 116 ++++ .../transforms/remove_unused_definitions.py | 23 + .../transforms/remove_unused_imports.py | 42 ++ .../transforms/remove_unused_variables.py | 34 ++ .../transforms/scope_analysis.py | 209 +++++++ .../transforms/side_effects.py | 129 +++++ test/test_dead_code.py | 209 +++++++ test/test_remove_unreachable.py | 522 ++++++++++++++++++ test/test_remove_unused_definitions.py | 297 ++++++++++ test/test_remove_unused_imports.py | 295 ++++++++++ test/test_remove_unused_variables.py | 364 ++++++++++++ 25 files changed, 2617 insertions(+), 2 deletions(-) create mode 100644 docs/source/minification_options/remove_unreachable.py create mode 100644 docs/source/minification_options/remove_unreachable.rst create mode 100644 docs/source/minification_options/remove_unused_definitions.py create mode 100644 docs/source/minification_options/remove_unused_definitions.rst create mode 100644 docs/source/minification_options/remove_unused_imports.py create mode 100644 docs/source/minification_options/remove_unused_imports.rst create mode 100644 docs/source/minification_options/remove_unused_variables.py create mode 100644 docs/source/minification_options/remove_unused_variables.rst create mode 100644 src/python_minifier/transforms/dead_code.py create mode 100644 src/python_minifier/transforms/remove_unreachable.py create mode 100644 src/python_minifier/transforms/remove_unused_definitions.py create mode 100644 src/python_minifier/transforms/remove_unused_imports.py create mode 100644 src/python_minifier/transforms/remove_unused_variables.py create mode 100644 src/python_minifier/transforms/scope_analysis.py create mode 100644 src/python_minifier/transforms/side_effects.py create mode 100644 test/test_dead_code.py create mode 100644 test/test_remove_unreachable.py create mode 100644 test/test_remove_unused_definitions.py create mode 100644 test/test_remove_unused_imports.py create mode 100644 test/test_remove_unused_variables.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ee1c1792..a68aef8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ and will output source code compatible with the version of the interpreter it is This means that if you minify code written for Python 3.11 using python-minifier running with Python 3.12, the minified code may only run with Python 3.12. +## [Unreleased] + +### Added +- New transforms to remove dead code. They are all disabled by default, and repeat until there is nothing left to remove: + + Remove unused imports, enabled with the `--remove-unused-imports` option. + + Remove assignments to unused variables, enabled with the `--remove-unused-variables` option. + + Remove unused function and class definitions, enabled with the `--remove-unused-definitions` option. + + Remove statements that follow a `return`, `raise`, `break` or `continue`, enabled with the `--remove-unreachable` option. + ## [3.4.0] - 2026-09-29 ### Added diff --git a/docs/source/minification_options/index.rst b/docs/source/minification_options/index.rst index e19b2eab..0e2d8457 100644 --- a/docs/source/minification_options/index.rst +++ b/docs/source/minification_options/index.rst @@ -31,4 +31,8 @@ They can be enabled or disabled through the minify function, or passing options rename_globals remove_asserts remove_debug + remove_unused_imports + remove_unused_variables + remove_unused_definitions + remove_unreachable prefer_single_line diff --git a/docs/source/minification_options/remove_unreachable.py b/docs/source/minification_options/remove_unreachable.py new file mode 100644 index 00000000..08ad2301 --- /dev/null +++ b/docs/source/minification_options/remove_unreachable.py @@ -0,0 +1,24 @@ +def check(value): + if value < 0: + raise ValueError(value) + print('never runs') + + if value: + return 'set' + else: + return 'unset' + + print('never runs') + + +def first(items): + for item in items: + if item: + return item + print('runs if there are no items') + return None + + +def generator(): + return + yield diff --git a/docs/source/minification_options/remove_unreachable.rst b/docs/source/minification_options/remove_unreachable.rst new file mode 100644 index 00000000..683e1bcc --- /dev/null +++ b/docs/source/minification_options/remove_unreachable.rst @@ -0,0 +1,38 @@ +Remove Unreachable +================== + +This transform removes statements that follow a statement that always leaves the block it is in. + +The statements after a ``return``, ``raise``, ``break`` or ``continue`` in the same block are removed, as they can never +run. The same is true for the statements after an ``if`` statement where every branch ends this way, so it needs an +``else`` clause. A ``try`` statement ends the block if its ``finally`` clause does, or if its body and all of its +``except`` clauses do. A ``with`` statement ends the block if its body does, unless it only does so by raising +an exception, as the context manager may suppress that. ``for`` and ``while`` loops never end the block, as their body +might not run. + +The statements are kept if removing them would change the meaning of the program, for example if they contain a ``yield``, +a ``global`` or ``nonlocal`` declaration, or the only assignment to a local variable that is used before it. + +If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used +in the module, nothing is removed. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. For example the names that were only used by unreachable statements may be removed +by :doc:`remove_unused_imports`. + +The transform is disabled by default. Enable it by passing the ``remove_unreachable=True`` argument to the :func:`python_minifier.minify` function, +or passing ``--remove-unreachable`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unreachable.py + +Output +~~~~~~ + +.. literalinclude:: remove_unreachable.min.py + :language: python diff --git a/docs/source/minification_options/remove_unused_definitions.py b/docs/source/minification_options/remove_unused_definitions.py new file mode 100644 index 00000000..61076c14 --- /dev/null +++ b/docs/source/minification_options/remove_unused_definitions.py @@ -0,0 +1,25 @@ +def helper(): + return 1 + + +def unused(): + return helper() + + +class Unused: + def method(self): + return 1 + + +@register +def registered(): + pass + + +def main(): + def inner(): + pass + return 2 + + +print(main()) diff --git a/docs/source/minification_options/remove_unused_definitions.rst b/docs/source/minification_options/remove_unused_definitions.rst new file mode 100644 index 00000000..e7a47633 --- /dev/null +++ b/docs/source/minification_options/remove_unused_definitions.rst @@ -0,0 +1,47 @@ +Remove Unused Definitions +========================= + +This transform removes functions and classes that are never used. + +A function or class definition is removed if its name is not used anywhere in the module. Methods and the other names +defined in a class body are never removed, only functions and classes defined at module level or inside a function. + +A definition is kept if executing it could have an effect, which is when: + + - It has a decorator, as the decorator may register the function or class somewhere + - A default value, annotation, base class or keyword evaluates something that isn't known to be harmless, like a call + - The body of a class runs anything that isn't a simple assignment, a docstring or a definition + +This is not always safe, so the transform is disabled by default. It could break any program that imports a +definition from the minified module, or looks up definitions by a string name. A class without a decorator could +still be registered by a base class or metaclass. + +Definitions are never removed: + + - If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used + in the module + - If the name is a dunder name like ``__getattr__`` + - If the name is included as a literal string in ``__all__`` at module level, or ``__all__`` is used in a way that + can't be determined by looking at the module + +If a definition is removed and a statement is still required, it is replaced by a zero expression statement. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. For example removing a function also removes any helper that only that function used. + +Enable this source transformation by passing the ``remove_unused_definitions=True`` argument to the :func:`python_minifier.minify` +function, or passing ``--remove-unused-definitions`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unused_definitions.py + +Output +~~~~~~ + +.. literalinclude:: remove_unused_definitions.min.py + :language: python diff --git a/docs/source/minification_options/remove_unused_imports.py b/docs/source/minification_options/remove_unused_imports.py new file mode 100644 index 00000000..ece44705 --- /dev/null +++ b/docs/source/minification_options/remove_unused_imports.py @@ -0,0 +1,12 @@ +import os +import sys, json +from collections import OrderedDict, defaultdict +from os import path as os_path + +print(sys.argv) +print(defaultdict(list)) + + +def listing(): + import glob + return [] diff --git a/docs/source/minification_options/remove_unused_imports.rst b/docs/source/minification_options/remove_unused_imports.rst new file mode 100644 index 00000000..802ca854 --- /dev/null +++ b/docs/source/minification_options/remove_unused_imports.rst @@ -0,0 +1,46 @@ +Remove Unused Imports +===================== + +This transform removes imports of names that are never used. + +An import statement is removed if none of the names it binds are used. If only some of the names are unused, just those +names are removed from the statement. A name is used if it is read anywhere in the module, including in nested +functions, decorators and default arguments. + +This is not always safe, so the transform is disabled by default. It could break: + + - Any program that imports a name from the minified module, as nothing is known about how the module is used elsewhere + - Imports that are only used for their side effects, like registering a codec + - Names that are only referred to by strings, for example type annotations that are string literals or code run by ``getattr()`` + +Imports are never removed: + + - If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used + in the module + - If they are ``__future__`` imports + - If the name is included as a literal string in ``__all__`` at module level, or ``__all__`` is used in a way that + can't be determined by looking at the module + - If they are in a class body, where the names become attributes of the class + +If an import statement is removed and a statement is still required, it is replaced by a zero expression statement. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. For example if :doc:`remove_unused_definitions` removes a function, the imports that +were only used by that function are removed too. + +Enable this source transformation by passing the ``remove_unused_imports=True`` argument to the :func:`python_minifier.minify` +function, or passing ``--remove-unused-imports`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unused_imports.py + +Output +~~~~~~ + +.. literalinclude:: remove_unused_imports.min.py + :language: python diff --git a/docs/source/minification_options/remove_unused_variables.py b/docs/source/minification_options/remove_unused_variables.py new file mode 100644 index 00000000..ad409a34 --- /dev/null +++ b/docs/source/minification_options/remove_unused_variables.py @@ -0,0 +1,18 @@ +TIMEOUT = 30 +RETRIES = TIMEOUT * 2 +VERSION = '1.0' +__version__ = VERSION + +started = start() + + +def process(items): + total = 0 + scratch = [item for item in items] + lookup = {'a': 1} + for item in items: + total += item + return total + + +print(RETRIES) diff --git a/docs/source/minification_options/remove_unused_variables.rst b/docs/source/minification_options/remove_unused_variables.rst new file mode 100644 index 00000000..45ffa3fb --- /dev/null +++ b/docs/source/minification_options/remove_unused_variables.rst @@ -0,0 +1,48 @@ +Remove Unused Variables +======================= + +This transform removes assignments to variables that are never used. + +An assignment is removed if the name is not used anywhere in the module, and evaluating the assigned value is known not +to have any effect. Assigning a literal, a collection of literals, a name, an attribute, a subscript, a lambda, +or an expression made of operators with these is removed. Anything that calls a function, uses ``await`` or ``yield``, +or is a comprehension is kept, as is any value that might unpack an iterable or mapping. + +Only assignments that bind a single name are removed, including annotated assignments with a value. +Assignments to attributes and subscripts, to multiple targets, and augmented assignments are always kept. +Using a name with ``del``, an augmented assignment or ``global`` and ``nonlocal`` declarations all count as a use. + +This is not always safe, so the transform is disabled by default. It could break any program that imports a +variable from the minified module, or reads variables by a string name. Reading an attribute or subscript, and applying +an operator, are assumed to have no effect, but they may run code that does. + +Assignments are never removed: + + - If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used + in the module + - If the name is a dunder name like ``__version__`` + - If the name is included as a literal string in ``__all__`` at module level, or ``__all__`` is used in a way that + can't be determined by looking at the module + - If they are in a class body, where the names become attributes of the class + +If an assignment is removed and a statement is still required, it is replaced by a zero expression statement. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. + +Enable this source transformation by passing the ``remove_unused_variables=True`` argument to the :func:`python_minifier.minify` +function, or passing ``--remove-unused-variables`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unused_variables.py + +Output +~~~~~~ + +.. literalinclude:: remove_unused_variables.min.py + :language: python diff --git a/src/python_minifier/__init__.py b/src/python_minifier/__init__.py index 8b091d2b..88aed9ea 100644 --- a/src/python_minifier/__init__.py +++ b/src/python_minifier/__init__.py @@ -22,6 +22,7 @@ ) from python_minifier.transforms.combine_imports import CombineImports from python_minifier.transforms.constant_folding import FoldConstants +from python_minifier.transforms.dead_code import eliminate_dead_code from python_minifier.transforms.remove_annotations import RemoveAnnotations from python_minifier.transforms.remove_annotations_options import RemoveAnnotationsOptions from python_minifier.transforms.remove_asserts import RemoveAsserts @@ -75,7 +76,11 @@ def minify( remove_builtin_exception_brackets=True, constant_folding=True, prefer_single_line=False, - remove_dead_branches=True + remove_dead_branches=True, + remove_unused_imports=False, + remove_unused_variables=False, + remove_unused_definitions=False, + remove_unreachable=False ): """ Minify a python module @@ -112,6 +117,10 @@ def minify( :param bool constant_folding: If literal expressions should be evaluated :param bool prefer_single_line: If semi-colons should be preferred over newlines where there is no difference in output size :param bool remove_dead_branches: If if-statements with a constant False test should be removed + :param bool remove_unused_imports: If imported names that are never used should be removed + :param bool remove_unused_variables: If assignments to variables that are never used should be removed + :param bool remove_unused_definitions: If functions and classes that are never used should be removed + :param bool remove_unreachable: If statements that follow a return, raise, break or continue should be removed :rtype: str @@ -164,6 +173,15 @@ def minify( if remove_dead_branches: module = RemoveDeadBranches()(module) + if remove_unused_imports or remove_unused_variables or remove_unused_definitions or remove_unreachable: + module = eliminate_dead_code( + module, + remove_imports=remove_unused_imports, + remove_variables=remove_unused_variables, + remove_definitions=remove_unused_definitions, + remove_unreachable=remove_unreachable + ) + if remove_explicit_return_none: module = RemoveExplicitReturnNone()(module) diff --git a/src/python_minifier/__init__.pyi b/src/python_minifier/__init__.pyi index fde0d748..50c596f9 100644 --- a/src/python_minifier/__init__.pyi +++ b/src/python_minifier/__init__.pyi @@ -30,7 +30,11 @@ def minify( remove_builtin_exception_brackets: bool = ..., constant_folding: bool = ..., prefer_single_line: bool = ..., - remove_dead_branches: bool = ... + remove_dead_branches: bool = ..., + remove_unused_imports: bool = ..., + remove_unused_variables: bool = ..., + remove_unused_definitions: bool = ..., + remove_unreachable: bool = ... ) -> Text: ... diff --git a/src/python_minifier/__main__.py b/src/python_minifier/__main__.py index bd524047..79f582b3 100644 --- a/src/python_minifier/__main__.py +++ b/src/python_minifier/__main__.py @@ -255,6 +255,30 @@ def parse_args(): help='Disable removing branches that are never executed', dest='remove_dead_branches', ) + minification_options.add_argument( + '--remove-unused-imports', + action='store_true', + help='Enable removing imports of names that are never used', + dest='remove_unused_imports', + ) + minification_options.add_argument( + '--remove-unused-variables', + action='store_true', + help='Enable removing assignments to variables that are never used', + dest='remove_unused_variables', + ) + minification_options.add_argument( + '--remove-unused-definitions', + action='store_true', + help='Enable removing functions and classes that are never used', + dest='remove_unused_definitions', + ) + minification_options.add_argument( + '--remove-unreachable', + action='store_true', + help='Enable removing statements that follow a return, raise, break or continue', + dest='remove_unreachable', + ) annotation_options = parser.add_argument_group('remove annotations options', 'Options that affect how annotations are removed') annotation_options.add_argument( @@ -391,6 +415,10 @@ def do_minify(source, filename, minification_args): remove_builtin_exception_brackets=minification_args.remove_exception_brackets, constant_folding=minification_args.constant_folding, prefer_single_line=minification_args.prefer_single_line, + remove_unused_imports=minification_args.remove_unused_imports, + remove_unused_variables=minification_args.remove_unused_variables, + remove_unused_definitions=minification_args.remove_unused_definitions, + remove_unreachable=minification_args.remove_unreachable, ) # Encode minified result to bytes for comparison and output diff --git a/src/python_minifier/transforms/dead_code.py b/src/python_minifier/transforms/dead_code.py new file mode 100644 index 00000000..04bd9257 --- /dev/null +++ b/src/python_minifier/transforms/dead_code.py @@ -0,0 +1,54 @@ +from python_minifier.rename import add_namespace, bind_names, resolve_names +from python_minifier.transforms.remove_unreachable import RemoveUnreachable +from python_minifier.transforms.remove_unused_definitions import RemoveUnusedDefinitions +from python_minifier.transforms.remove_unused_imports import RemoveUnusedImports +from python_minifier.transforms.remove_unused_variables import RemoveUnusedVariables + + +def eliminate_dead_code(module, remove_imports, remove_variables, remove_definitions, remove_unreachable): + """ + Remove code that has no effect from a module + + Removing code can make more code dead, for example removing an unused function can leave the names + it used unused. This repeats until there is nothing left to remove. + + Modules that are tainted, because they use features like star imports or ``exec``, are not changed. + + The names in the module are left unbound on return. + + :param module: The module to remove dead code from + :type module: ast.Module + :param bool remove_imports: If unused imports should be removed + :param bool remove_variables: If assignments to unused variables should be removed + :param bool remove_definitions: If unused function and class definitions should be removed + :param bool remove_unreachable: If statements after return, raise, break or continue should be removed + :rtype: ast.Module + """ + + transforms = [] + if remove_unreachable: + transforms.append(RemoveUnreachable()) + if remove_definitions: + transforms.append(RemoveUnusedDefinitions()) + if remove_variables: + transforms.append(RemoveUnusedVariables()) + if remove_imports: + transforms.append(RemoveUnusedImports()) + + changed = True + while changed: + # Names must be bound again to account for what was removed in the last pass + add_namespace(module) + bind_names(module) + resolve_names(module) + + if module.tainted: + break + + changed = False + for transform in transforms: + module = transform(module) + changed = changed or transform.changed + + add_namespace(module) + return module diff --git a/src/python_minifier/transforms/remove_unreachable.py b/src/python_minifier/transforms/remove_unreachable.py new file mode 100644 index 00000000..bab5dbe0 --- /dev/null +++ b/src/python_minifier/transforms/remove_unreachable.py @@ -0,0 +1,116 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.remove_dead_branches import in_function_scope +from python_minifier.transforms.scope_analysis import DeadCodeTransformer, makes_local + +_TRY = tuple(getattr(ast, name) for name in ('Try', 'TryExcept', 'TryFinally') if hasattr(ast, name)) + + +def _suite_terminates(suite, raises): + return bool(suite) and _terminates(suite[-1], raises) + + +def _terminates(node, raises=True): + """ + Does control never continue to the statement after this one? + + :param node: The statement to check + :type node: ast.AST + :param bool raises: If a ``raise`` statement counts as terminating + :rtype: bool + """ + + if isinstance(node, (ast.Return, ast.Break, ast.Continue)): + return True + + if isinstance(node, ast.Raise): + return raises + + if isinstance(node, ast.If): + return bool(node.orelse) and _suite_terminates(node.body, raises) and _suite_terminates(node.orelse, raises) + + if isinstance(node, ast.TryStar): + return _suite_terminates(node.finalbody, raises) + + if isinstance(node, _TRY): + if _suite_terminates(getattr(node, 'finalbody', []), raises): + return True + + if not (_suite_terminates(node.body, raises) or _suite_terminates(getattr(node, 'orelse', []), raises)): + return False + + return all(_suite_terminates(handler.body, raises) for handler in getattr(node, 'handlers', [])) + + if isinstance(node, (ast.With, ast.AsyncWith)): + # The context manager can suppress an exception raised in the body + return _suite_terminates(node.body, raises=False) + + # Loops may never run their body, and the else clause of a loop runs when it is not broken out of + return False + + +class RemoveUnreachable(DeadCodeTransformer): + """ + Remove statements that follow a statement that always return, raises, breaks or continues + + Statements are kept if removing them would change the meaning of the program, for example if + they contain a ``yield`` or the only assignment to a local variable that is used earlier. + """ + + def suite(self, node_list, parent): + reachable = [] + + for index, node in enumerate(node_list): + reachable.append(self.visit(node)) + + unreachable = node_list[index + 1:] + if unreachable and _terminates(node) and self.can_remove(unreachable): + self.changed = True + break + + return reachable + + def can_remove(self, statements): + """ + Can these statements be removed without changing the meaning of the enclosing scope? + + :param list statements: Consecutive statements in a suite + :rtype: bool + """ + + namespace = statements[0].namespace + nodes = [node for statement in statements for node in ast.walk(statement)] + + for node in nodes: + if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Global, ast.Nonlocal)) and getattr(node, 'namespace', None) is namespace: + return False + + if isinstance(namespace, ast.Module) or (isinstance(namespace, ast.ClassDef) and not in_function_scope(namespace)): + # Name resolution is dynamic in these scopes + return True + + removed = {id(node) for node in nodes} + local_bindings = {id(binding) for binding in namespace.bindings} + + for node in nodes: + if getattr(node, 'namespace', None) is not namespace or not makes_local(node): + continue + + binding = self.scope.binding(node) + if binding is None: + continue + + if isinstance(namespace, ast.ClassDef): + # A class body binding hides the enclosing function's name from loads in the class body only + references = [reference for reference in binding.references if getattr(reference, 'namespace', None) is namespace] + elif id(binding) in local_bindings: + references = binding.references + else: + continue + + # Without these statements the name would no longer be local to this namespace + remaining = [reference for reference in references if id(reference) not in removed] + if remaining and not any(makes_local(reference) for reference in remaining): + return False + + return True diff --git a/src/python_minifier/transforms/remove_unused_definitions.py b/src/python_minifier/transforms/remove_unused_definitions.py new file mode 100644 index 00000000..71ee19dd --- /dev/null +++ b/src/python_minifier/transforms/remove_unused_definitions.py @@ -0,0 +1,23 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.scope_analysis import DeadCodeTransformer +from python_minifier.transforms.side_effects import is_definition_side_effect_free + + +class RemoveUnusedDefinitions(DeadCodeTransformer): + """ + Remove function and class definitions that are never used + + Decorated definitions are never removed, as the decorator may register the definition somewhere. + Neither are definitions that evaluate something with side effects when they are executed. + """ + + def eliminate(self, node): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return node + + if not is_definition_side_effect_free(node) or not self.scope.is_removable(node, node.name): + return node + + self.changed = True + return None diff --git a/src/python_minifier/transforms/remove_unused_imports.py b/src/python_minifier/transforms/remove_unused_imports.py new file mode 100644 index 00000000..f9e8f423 --- /dev/null +++ b/src/python_minifier/transforms/remove_unused_imports.py @@ -0,0 +1,42 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.scope_analysis import DeadCodeTransformer + + +class RemoveUnusedImports(DeadCodeTransformer): + """ + Remove imported names that are never used + + An import statement is removed when none of the names it binds are used. If only some of them are unused, + only those names are removed from the statement. + + ``__future__`` imports are never removed. + """ + + def eliminate(self, node): + if not isinstance(node, (ast.Import, ast.ImportFrom)): + return node + + if isinstance(node, ast.ImportFrom) and node.module == '__future__': + return node + + used = [alias for alias in node.names if not self.is_unused(alias)] + if len(used) == len(node.names): + return node + + self.changed = True + + if not used: + return None + + node.names = used + return node + + def is_unused(self, alias): + if alias.name == '*': + return False + + # A dotted import without an alias binds only the root package + name = alias.asname or alias.name.split('.')[0] + + return self.scope.is_removable(alias, name) diff --git a/src/python_minifier/transforms/remove_unused_variables.py b/src/python_minifier/transforms/remove_unused_variables.py new file mode 100644 index 00000000..e4a969ed --- /dev/null +++ b/src/python_minifier/transforms/remove_unused_variables.py @@ -0,0 +1,34 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.scope_analysis import DeadCodeTransformer +from python_minifier.transforms.side_effects import is_side_effect_free + + +class RemoveUnusedVariables(DeadCodeTransformer): + """ + Remove assignments to variables that are never used + + Only assignments of a single name are removed, and only when the assigned value is known to have no side effects. + """ + + def eliminate(self, node): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + annotation = None + elif isinstance(node, ast.AnnAssign) and node.value is not None: + target = node.target + annotation = node.annotation + else: + return node + + if not isinstance(target, ast.Name): + return node + + if not is_side_effect_free(node.value) or not is_side_effect_free(annotation): + return node + + if not self.scope.is_removable(target, target.id): + return node + + self.changed = True + return None diff --git a/src/python_minifier/transforms/scope_analysis.py b/src/python_minifier/transforms/scope_analysis.py new file mode 100644 index 00000000..b5ddb8a6 --- /dev/null +++ b/src/python_minifier/transforms/scope_analysis.py @@ -0,0 +1,209 @@ +import python_minifier.ast_compat as ast +from python_minifier.ast_annotation import get_parent +from python_minifier.transforms.suite_transformer import SuiteTransformer +from python_minifier.util import is_constant_node + + +def is_dunder_name(name): + """ + Is this a name like __init__, which has a special meaning to python + + :param str name: The name to check + :rtype: bool + """ + + return name.startswith('__') and name.endswith('__') + + +def reads_name(reference): + """ + Does a reference to a binding use the current value of the name? + + A binding that is only referenced by the statements that define it is never used. + Anything else that mentions the name counts as a use, including ``del`` statements, + augmented assignments and ``global`` or ``nonlocal`` declarations, which let other + scopes read and write the name. + + :param reference: A node from the references of a binding + :type reference: ast.AST + :rtype: bool + """ + + if isinstance(reference, ast.Name): + if isinstance(reference.ctx, (ast.Load, ast.Del)): + return True + + if isinstance(reference.ctx, ast.Store): + try: + return isinstance(get_parent(reference), ast.AugAssign) + except ValueError: + return True + + return False + + return isinstance(reference, (ast.Global, ast.Nonlocal)) + + +def makes_local(reference): + """ + Does a reference to a binding cause the name to be local to its namespace? + + :param reference: A node from the references of a binding + :type reference: ast.AST + :rtype: bool + """ + + if isinstance(reference, ast.Name): + return not isinstance(reference.ctx, ast.Load) + + return not isinstance(reference, (ast.Global, ast.Nonlocal)) + + +def find_exports(module): + """ + Find the names a module lists in ``__all__`` + + :param module: The module to search + :type module: ast.Module + :return: The listed names, or None if ``__all__`` is used in a way that can't be determined statically + :rtype: set[str] or None + """ + + names = set() + + for node in ast.walk(module): + if isinstance(node, ast.Name) and node.id == '__all__' and isinstance(node.ctx, ast.Load): + return None + + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): + targets = [node.target] + else: + continue + + if not any(isinstance(target, ast.Name) and target.id == '__all__' for target in targets): + continue + + if node.value is None: + continue + + if not isinstance(node.value, (ast.List, ast.Tuple, ast.Set)): + return None + + for element in node.value.elts: + if not is_constant_node(element, ast.Str): + return None + names.add(element.s) + + return names + + +class ScopeAnalysis(object): + """ + Answers questions about how names are used in a module + + The module must have had its names bound and resolved, the analysis is invalid once + the module is changed. + + :param module: The module to analyse + :type module: ast.Module + """ + + def __init__(self, module): + self._binding_for_reference = {} + + for node in ast.walk(module): + for binding in getattr(node, 'bindings', ()): + for reference in binding.references: + self._binding_for_reference[id(reference)] = binding + + self._exports = find_exports(module) + + def binding(self, node): + """ + The binding that a node creates or references + + :type node: ast.AST + :rtype: python_minifier.rename.binding.NameBinding or None + """ + + return self._binding_for_reference.get(id(node)) + + def is_removable(self, node, name): + """ + Can the statement that makes a node bind a name be removed, as far as the name is concerned? + + The name must never be used in the module. Names that could be used from outside the + module, or that the interpreter gives a meaning to, are never unused. Class namespaces + are skipped as the names in them become attributes of the class. + + :param node: The node that binds the name. This is a Name, alias, FunctionDef, AsyncFunctionDef or ClassDef + :type node: ast.AST + :param str name: The name being bound + :rtype: bool + """ + + namespace = node.namespace + + if isinstance(namespace, ast.ClassDef) or is_dunder_name(name): + return False + + if isinstance(namespace, ast.Module) and (self._exports is None or name in self._exports): + return False + + binding = self.binding(node) + if binding is None: + return False + + return not any(reads_name(reference) for reference in binding.references) + + +class DeadCodeTransformer(SuiteTransformer): + """ + Base class for transforms that remove statements that have no effect on a bound module + + The :attr:`changed` attribute is set if the last run removed anything. + """ + + def __call__(self, module): + self.changed = False + self.scope = ScopeAnalysis(module) + return self.visit(module) + + def eliminate(self, node): + """ + Remove a statement, or part of it + + :param node: The statement to consider + :type node: ast.AST + :return: The statement to keep, or None if it should be removed + :rtype: ast.AST or None + """ + + return node + + def suite(self, node_list, parent): + without_dead_code = [] + + for node in node_list: + node = self.eliminate(node) + if node is not None: + without_dead_code.append(self.visit(node)) + + return self.non_empty(without_dead_code, parent) + + def non_empty(self, suite, parent): + """ + Ensure a suite has a statement, if it needs one + + :param list suite: The statements in the suite + :param parent: The node that owns the suite + :type parent: ast.AST + :rtype: list + """ + + if suite or isinstance(parent, ast.Module): + return suite + + return [self.add_child(ast.Expr(value=ast.Num(0)), parent=parent)] diff --git a/src/python_minifier/transforms/side_effects.py b/src/python_minifier/transforms/side_effects.py new file mode 100644 index 00000000..b514828b --- /dev/null +++ b/src/python_minifier/transforms/side_effects.py @@ -0,0 +1,129 @@ +import python_minifier.ast_compat as ast +from python_minifier.util import is_constant_node + +_FSTRINGS = tuple(getattr(ast, name) for name in ('JoinedStr', 'FormattedValue') if hasattr(ast, name)) + + +def is_side_effect_free(node): + """ + Is evaluating an expression, and discarding the result, known to do nothing? + + The analysis is conservative, anything that is not known to be harmless is assumed to + have side effects. Calls, ``await``, ``yield``, assignment expressions and + comprehensions are never side effect free. Looking up a name, accessing an + attribute or subscript and applying an operator are assumed not to run code that + matters, and so are not considered to have side effects. + + :param node: The expression to check, or None + :type node: ast.AST or None + :rtype: bool + """ + + if node is None: + return True + + if is_constant_node(node, (ast.Num, ast.Str, ast.Bytes, ast.NameConstant, ast.Ellipsis)) or isinstance(node, ast.Name): + return True + + if isinstance(node, (ast.Tuple, ast.List, ast.Set)): + return all(is_side_effect_free(element) for element in node.elts) + + if isinstance(node, ast.Dict): + return all(key is not None and is_side_effect_free(key) for key in node.keys) and all(is_side_effect_free(value) for value in node.values) + + if isinstance(node, ast.UnaryOp): + return is_side_effect_free(node.operand) + + if isinstance(node, ast.BinOp): + return is_side_effect_free(node.left) and is_side_effect_free(node.right) + + if isinstance(node, ast.BoolOp): + return all(is_side_effect_free(value) for value in node.values) + + if isinstance(node, ast.Compare): + return is_side_effect_free(node.left) and all(is_side_effect_free(comparator) for comparator in node.comparators) + + if isinstance(node, ast.IfExp): + return is_side_effect_free(node.test) and is_side_effect_free(node.body) and is_side_effect_free(node.orelse) + + if isinstance(node, ast.Attribute): + return is_side_effect_free(node.value) + + if isinstance(node, ast.Subscript): + return is_side_effect_free(node.value) and is_side_effect_free(node.slice) + + if isinstance(node, ast.Slice): + return is_side_effect_free(node.lower) and is_side_effect_free(node.upper) and is_side_effect_free(node.step) + + if isinstance(node, ast.Index): + return is_side_effect_free(node.value) + + if isinstance(node, ast.ExtSlice): + return all(is_side_effect_free(dim) for dim in node.dims) + + if isinstance(node, ast.Lambda): + return _are_defaults_side_effect_free(node.args) + + if isinstance(node, _FSTRINGS): + if hasattr(node, 'values'): + return all(is_side_effect_free(value) for value in node.values) + return is_side_effect_free(node.value) and is_side_effect_free(node.format_spec) + + return False + + +def _are_defaults_side_effect_free(arguments): + defaults = list(arguments.defaults) + list(getattr(arguments, 'kw_defaults', [])) + return all(is_side_effect_free(default) for default in defaults) + + +def is_definition_side_effect_free(node): + """ + Is executing a function or class definition, and discarding the result, known to do nothing? + + Defining a function evaluates its default values and annotations. Defining a class evaluates its + bases and keywords and runs its body. + Decorators are not considered, a decorated definition is never side effect free. + + :param node: The definition to check + :type node: ast.FunctionDef or ast.AsyncFunctionDef or ast.ClassDef + :rtype: bool + """ + + if node.decorator_list: + return False + + if isinstance(node, ast.ClassDef): + expressions = list(node.bases) + [keyword.value for keyword in getattr(node, 'keywords', [])] + expressions += [getattr(node, name) for name in ('starargs', 'kwargs') if getattr(node, name, None) is not None] + return all(is_side_effect_free(expression) for expression in expressions) and all(_is_class_statement_side_effect_free(statement) for statement in node.body) + + arguments = node.args + annotations = [getattr(node, 'returns', None)] + annotations += [getattr(arg, 'annotation', None) for arg in getattr(arguments, 'posonlyargs', []) + arguments.args + getattr(arguments, 'kwonlyargs', [])] + annotations += [getattr(arguments, name, None) for name in ('varargannotation', 'kwargannotation')] + if isinstance(arguments.vararg, ast.arg): + annotations.append(arguments.vararg.annotation) + if isinstance(arguments.kwarg, ast.arg): + annotations.append(arguments.kwarg.annotation) + + return _are_defaults_side_effect_free(arguments) and all(is_side_effect_free(annotation) for annotation in annotations) + + +def _is_class_statement_side_effect_free(statement): + if isinstance(statement, ast.Pass): + return True + + if isinstance(statement, ast.Expr): + return is_side_effect_free(statement.value) + + if isinstance(statement, ast.Assign): + return is_side_effect_free(statement.value) and all(isinstance(target, ast.Name) for target in statement.targets) + + if isinstance(statement, ast.AnnAssign): + return isinstance(statement.target, ast.Name) and is_side_effect_free(statement.annotation) and is_side_effect_free(statement.value) + + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return is_definition_side_effect_free(statement) + + return False diff --git a/test/test_dead_code.py b/test/test_dead_code.py new file mode 100644 index 00000000..c4def38b --- /dev/null +++ b/test/test_dead_code.py @@ -0,0 +1,209 @@ +import ast +import sys + +import pytest + +from python_minifier import minify +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code +from subprocess_compat import run_subprocess, safe_decode + +SOURCE = ''' +import os +import sys +import json + +unused_variable = 42 + +def unused_function(): + return json.dumps({}) + +class UnusedClass: + pass + +def main(): + unused_local = 1 + return sys.argv + print('unreachable') + +call(main()) +''' + + +def eliminate_all(source): + module = ast.parse(source, 'dead_code') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=True, remove_variables=True, remove_definitions=True, remove_unreachable=True) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = eliminate_all(source) + compare_ast(expected_ast, actual_ast) + + +def test_all_transforms_together(): + run_test(SOURCE, ''' +import sys + +def main(): + return sys.argv + +call(main()) +''') + + +def test_removing_a_function_makes_its_imports_unused(): + run_test(''' +import json +def helper(): + return json.dumps({}) +def main(): + return 1 +print(main()) +''', ''' +def main(): + return 1 +print(main()) +''') + + +def test_removing_unreachable_code_makes_names_unused(): + run_test(''' +import json +def helper(): + pass +def main(): + return 1 + return json.dumps(helper()) +print(main()) +''', ''' +def main(): + return 1 +print(main()) +''') + + +def test_removing_a_variable_makes_its_import_unused(): + run_test(''' +import json +value = json +print(1) +''', 'print(1)') + + +def test_values_with_side_effects_keep_their_imports(): + source = ''' +from os.path import join +path = join('a', 'b') +print(1) +''' + run_test(source, source) + + +def test_long_chain_of_dead_code(): + names = ['a%d' % i for i in range(20)] + source = '\n'.join('def %s():\n return %s()' % (name, following) for name, following in zip(names, names[1:])) + source += '\ndef %s():\n return 1\nprint(1)\n' % names[-1] + run_test(source, 'print(1)') + + +def test_nothing_to_remove(): + source = ''' +import sys +def main(): + return sys.argv +print(main()) +''' + run_test(source, source) + + +def test_tainted_module_is_not_changed(): + for source in ['from os import *\n' + SOURCE, SOURCE + "\neval('1')\n", 'import timeit\n' + SOURCE]: + expected_ast = ast.parse(source) + actual_ast = eliminate_all(source) + compare_ast(expected_ast, actual_ast) + + +def test_options_are_disabled_by_default(): + assert minify(SOURCE) == minify(SOURCE, remove_unused_imports=False, remove_unused_variables=False, remove_unused_definitions=False, remove_unreachable=False) + assert 'unused_function' in minify(SOURCE) + assert 'unused_variable' in minify(SOURCE) + assert 'UnusedClass' in minify(SOURCE) + assert 'unreachable' in minify(SOURCE) + assert 'import os' in minify(SOURCE) + + +def test_minify_options_are_independent(): + assert 'import os' not in minify(SOURCE, remove_unused_imports=True) + assert 'unused_function' in minify(SOURCE, remove_unused_imports=True) + + assert 'unused_variable' not in minify(SOURCE, remove_unused_variables=True) + assert 'import os' in minify(SOURCE, remove_unused_variables=True) + + assert 'unused_function' not in minify(SOURCE, remove_unused_definitions=True) + assert 'UnusedClass' not in minify(SOURCE, remove_unused_definitions=True) + assert 'unused_variable' in minify(SOURCE, remove_unused_definitions=True) + + assert 'unreachable' not in minify(SOURCE, remove_unreachable=True) + assert 'unused_function' in minify(SOURCE, remove_unreachable=True) + + +def test_minify_removes_everything_dead(): + minified = minify(SOURCE, remove_unused_imports=True, remove_unused_variables=True, remove_unused_definitions=True, remove_unreachable=True) + assert minified == 'import sys\ndef main():return sys.argv\ncall(main())' + + +def test_minify_with_renaming(): + minified = minify(SOURCE, remove_unused_imports=True, remove_unused_variables=True, remove_unused_definitions=True, remove_unreachable=True, rename_globals=True) + assert minified == 'import sys\ndef A():return sys.argv\ncall(A())' + + +def test_minify_after_removing_asserts(): + source = ''' +import os +def main(): + assert os + return 1 +print(main()) +''' + assert 'os' in minify(source, remove_unused_imports=True) + assert 'os' not in minify(source, remove_asserts=True, remove_unused_imports=True) + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='annotations need python 3') +def test_minify_with_annotations(): + source = ''' +import typing +def f(a: typing.List) -> None: + pass +print(f) +''' + assert 'typing' in minify(source, remove_annotations=False, remove_unused_imports=True) + assert 'typing' not in minify(source, remove_unused_imports=True) + + +def test_minify_leaves_tainted_module_alone(): + source = 'from os import *\nimport sys\nx = 1\ncall(1)' + assert minify(source, remove_unused_imports=True, remove_unused_variables=True, remove_unused_definitions=True, remove_unreachable=True) == 'from os import*\nimport sys\nx=1\ncall(1)' + + +def pyminify(*args): + result = run_subprocess([sys.executable, '-m', 'python_minifier', '-'] + list(args), timeout=30, input_data=SOURCE) + assert result.returncode == 0 + return safe_decode(result.stdout).replace('\r\n', '\n') + + +def test_cli_options_are_disabled_by_default(): + assert pyminify() == minify(SOURCE) + + +def test_cli_options(): + assert pyminify('--remove-unused-imports') == minify(SOURCE, remove_unused_imports=True) + assert pyminify('--remove-unused-variables') == minify(SOURCE, remove_unused_variables=True) + assert pyminify('--remove-unused-definitions') == minify(SOURCE, remove_unused_definitions=True) + assert pyminify('--remove-unreachable') == minify(SOURCE, remove_unreachable=True) diff --git a/test/test_remove_unreachable.py b/test/test_remove_unreachable.py new file mode 100644 index 00000000..6a580760 --- /dev/null +++ b/test/test_remove_unreachable.py @@ -0,0 +1,522 @@ +import ast +import sys + +import pytest + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unreachable(source): + module = ast.parse(source, 'remove_unreachable') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=False, remove_variables=False, remove_definitions=False, remove_unreachable=True) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unreachable(source) + compare_ast(expected_ast, actual_ast) + + +def test_after_return(): + run_test(''' +def f(): + return 1 + print('unreachable') +''', ''' +def f(): + return 1 +''') + + +def test_after_raise(): + run_test(''' +def f(): + raise ValueError() + print('unreachable') +''', ''' +def f(): + raise ValueError() +''') + + +def test_after_break_and_continue(): + run_test(''' +for i in range(10): + if i: + break + print('unreachable') + continue + print('unreachable') +else: + print('reachable') +print('reachable') +''', ''' +for i in range(10): + if i: + break + continue +else: + print('reachable') +print('reachable') +''') + + +def test_removes_all_following_statements(): + run_test(''' +def f(): + return 1 + x = 2 + def g(): + pass + class C: + pass + print(x) +''', ''' +def f(): + return 1 +''') + + +def test_module_level(): + run_test(''' +raise SystemExit(0) +print('unreachable') +''', 'raise SystemExit(0)') + + +def test_nested_suites(): + run_test(''' +def f(a): + while a: + if a: + return 1 + print('unreachable') + for i in a: + try: + raise ValueError + print('unreachable') + except ValueError: + continue + print('unreachable') + finally: + pass +''', ''' +def f(a): + while a: + if a: + return 1 + for i in a: + try: + raise ValueError + except ValueError: + continue + finally: + pass +''') + + +def test_in_class_body(): + run_test(''' +class C: + raise ValueError + x = 1 +''', ''' +class C: + raise ValueError +''') + + +def test_if_else_that_both_terminate(): + run_test(''' +def f(a): + if a: + return 1 + else: + return 2 + print('unreachable') +''', ''' +def f(a): + if a: + return 1 + else: + return 2 +''') + + run_test(''' +def f(a, b): + if a: + return 1 + elif b: + raise ValueError + else: + return 2 + print('unreachable') +''', ''' +def f(a, b): + if a: + return 1 + elif b: + raise ValueError + else: + return 2 +''') + + +def test_if_that_may_not_terminate(): + source = ''' +def f(a, b): + if a: + return 1 + print('reachable') + if a: + return 1 + else: + pass + print('reachable') + if a: + return 1 + elif b: + return 2 + print('reachable') +''' + run_test(source, source) + + +def test_try_with_handlers_that_all_terminate(): + run_test(''' +def f(): + try: + return 1 + except ValueError: + return 2 + except KeyError: + raise + print('unreachable') +''', ''' +def f(): + try: + return 1 + except ValueError: + return 2 + except KeyError: + raise +''') + + +def test_try_that_may_not_terminate(): + source = ''' +def f(): + try: + return 1 + except ValueError: + pass + print('reachable') + try: + pass + except ValueError: + return 2 + print('reachable') +''' + run_test(source, source) + + +def test_try_with_finally_that_terminates(): + run_test(''' +def f(): + try: + pass + finally: + return 1 + print('unreachable') +''', ''' +def f(): + try: + pass + finally: + return 1 +''') + + +def test_try_with_finally_that_does_not_terminate(): + source = ''' +def f(): + try: + pass + finally: + cleanup() + print('reachable') +''' + run_test(source, source) + + +def test_try_body_that_terminates_with_finally(): + run_test(''' +def f(): + try: + return 1 + finally: + cleanup() + print('unreachable') +''', ''' +def f(): + try: + return 1 + finally: + cleanup() +''') + + +def test_try_else_that_terminates(): + run_test(''' +def f(): + try: + pass + except ValueError: + return 1 + else: + return 2 + print('unreachable') +''', ''' +def f(): + try: + pass + except ValueError: + return 1 + else: + return 2 +''') + + +def test_with_that_returns(): + run_test(''' +def f(): + with open('f') as fp: + return fp.read() + print('unreachable') +''', ''' +def f(): + with open('f') as fp: + return fp.read() +''') + + +def test_with_that_raises_may_suppress_the_exception(): + source = ''' +def f(): + with suppress(ValueError): + raise ValueError + print('reachable') + with suppress(ValueError): + if a: + return 1 + else: + raise ValueError + print('reachable') +''' + run_test(source, source) + + +def test_loops_may_not_terminate(): + source = ''' +def f(a): + for i in a: + return 1 + print('reachable') + while a: + return 1 + print('reachable') + while True: + return 1 + print('reachable') +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason='match needs python 3.10') +def test_match_may_not_terminate(): + source = ''' +def f(a): + match a: + case 1: + return 1 + case _: + return 2 + print('reachable') +''' + run_test(source, source) + + +def test_yield_is_kept(): + source = ''' +def f(): + return + yield +''' + run_test(source, source) + + source = ''' +def f(): + return + x = yield +''' + run_test(source, source) + + +def test_yield_in_nested_function_is_removed(): + run_test(''' +def f(): + return + def g(): + yield +''', ''' +def f(): + return +''') + + +@pytest.mark.skipif(sys.version_info < (3, 3), reason='yield from needs python 3.3') +def test_yield_from_is_kept(): + source = ''' +def f(): + return + yield from g() +''' + run_test(source, source) + + +def test_global_declaration_is_kept(): + source = ''' +def f(): + return + global x + x = 1 +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='nonlocal needs python 3') +def test_nonlocal_declaration_is_kept(): + source = ''' +def f(): + x = 1 + def g(): + return + nonlocal x + return g +''' + run_test(source, source) + + +def test_only_assignment_to_a_local_name_is_kept(): + source = ''' +x = 1 +def f(): + return x + x = 2 +''' + run_test(source, source) + + source = ''' +x = 1 +def f(): + return x + def x(): + pass +''' + run_test(source, source) + + source = ''' +x = 1 +def f(): + return x + import x +''' + run_test(source, source) + + source = ''' +x = 1 +def f(): + def g(): + return x + return g + x = 2 +''' + run_test(source, source) + + +def test_local_name_that_is_not_used_otherwise_is_removed(): + run_test(''' +def f(): + return 1 + x = 2 + print(x) +''', ''' +def f(): + return 1 +''') + + +def test_local_name_that_is_still_bound_is_removed(): + run_test(''' +def f(): + x = 1 + return x + x = 2 +''', ''' +def f(): + x = 1 + return x +''') + + +def test_module_names_are_dynamic(): + run_test(''' +print(x) +raise SystemExit +x = 1 +''', ''' +print(x) +raise SystemExit +''') + + +def test_class_names_are_dynamic(): + run_test(''' +class C: + print(x) + raise ValueError + x = 1 +''', ''' +class C: + print(x) + raise ValueError +''') + + +def test_class_names_in_function_are_kept(): + source = ''' +def f(): + x = 1 + class C: + print(x) + raise ValueError + x = 2 + return C +''' + run_test(source, source) + + +def test_exec_taints_module(): + source = ''' +def f(): + return 1 + print('unreachable') +exec('f()') +''' + run_test(source, source) diff --git a/test/test_remove_unused_definitions.py b/test/test_remove_unused_definitions.py new file mode 100644 index 00000000..af85a86e --- /dev/null +++ b/test/test_remove_unused_definitions.py @@ -0,0 +1,297 @@ +import ast +import sys + +import pytest + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unused_definitions(source): + module = ast.parse(source, 'remove_unused_definitions') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=False, remove_variables=False, remove_definitions=True, remove_unreachable=False) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unused_definitions(source) + compare_ast(expected_ast, actual_ast) + + +def test_unused_function(): + run_test(''' +def unused(): + pass +print(1) +''', 'print(1)') + + +def test_unused_class(): + run_test(''' +class Unused: + pass +print(1) +''', 'print(1)') + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason='async functions need python 3.5') +def test_unused_async_function(): + run_test(''' +async def unused(): + pass +print(1) +''', 'print(1)') + + +def test_used_definitions(): + source = ''' +def used(): + return 1 +class Used: + pass +print(used(), Used()) +''' + run_test(source, source) + + +def test_function_passed_as_argument_is_used(): + source = ''' +def callback(): + pass +register(callback) +''' + run_test(source, source) + + +def test_class_used_as_base_is_used(): + source = ''' +class Base: + pass +class Child(Base): + pass +print(Child()) +''' + run_test(source, source) + + +def test_class_used_in_except_clause_is_used(): + source = ''' +class MyError(Exception): + pass +try: + pass +except MyError: + pass +''' + run_test(source, source) + + +def test_decorated_definitions_are_kept(): + source = ''' +@app.route('/') +def index(): + pass +@dataclass +class Config: + pass +''' + run_test(source, source) + + +def test_decorators_are_used(): + source = ''' +def decorator(f): + return f +@decorator +def main(): + pass +''' + run_test(source, source) + + +def test_dunder_definitions_are_kept(): + source = ''' +def __getattr__(name): + pass +class __Meta__: + pass +''' + run_test(source, source) + + +def test_definitions_listed_in_all_are_kept(): + run_test(''' +__all__ = ['helper', 'Helper'] +def helper(): + pass +class Helper: + pass +def other(): + pass +''', ''' +__all__ = ['helper', 'Helper'] +def helper(): + pass +class Helper: + pass +''') + + +def test_dynamic_all_keeps_every_module_definition(): + source = ''' +__all__ = names() +def helper(): + pass +''' + run_test(source, source) + + +def test_methods_are_kept(): + source = ''' +class C: + def method(self): + pass + class Inner: + pass +print(C) +''' + run_test(source, source) + + +def test_unused_function_in_function(): + run_test(''' +def outer(): + def inner(): + pass + return 1 +print(outer()) +''', ''' +def outer(): + return 1 +print(outer()) +''') + + +def test_used_function_in_function(): + source = ''' +def outer(): + def inner(): + pass + return inner +print(outer()) +''' + run_test(source, source) + + +def test_function_in_function_is_not_used_by_other_scopes(): + run_test(''' +def helper(): + pass +def outer(): + def helper(): + pass + return 1 +print(helper(), outer()) +''', ''' +def helper(): + pass +def outer(): + return 1 +print(helper(), outer()) +''') + + +def test_empty_suite_is_replaced(): + run_test(''' +def outer(): + def inner(): + pass +print(outer()) +''', ''' +def outer(): + 0 +print(outer()) +''') + + +def test_definitions_that_evaluate_calls_are_kept(): + source = ''' +def f(a=call()): + pass +class C(base()): + pass +class D: + attribute = call() +class E: + call() +''' + run_test(source, source) + + +def test_definitions_with_harmless_defaults_are_removed(): + run_test(''' +def f(a=1, *args, **kwargs): + call() +class C(Base): + x = 1 + y = 'text' + def method(self, a=2): + call() + class Inner: + pass +print(1) +''', 'print(1)') + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='keyword only arguments and metaclass keywords need python 3') +def test_definitions_with_harmless_keywords_are_removed(): + run_test(''' +def f(a, *args, b=[], **kwargs): + call() +class C(Base, metaclass=Meta): + pass +print(1) +''', 'print(1)') + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='annotations need python 3') +def test_definitions_with_annotations_that_evaluate_calls_are_kept(): + source = ''' +def f(a: call()): + pass +def g() -> call(): + pass +''' + run_test(source, source) + + run_test(''' +def f(a: int, *args: str, b: 'text' = 1, **kwargs: dict) -> None: + pass +print(1) +''', 'print(1)') + + +def test_removing_a_definition_can_make_another_unused(): + run_test(''' +def a(): + return b() +def b(): + return c() +def c(): + return 1 +print(1) +''', 'print(1)') + + +def test_exec_taints_module(): + source = ''' +def f(): + pass +exec('f()') +''' + run_test(source, source) diff --git a/test/test_remove_unused_imports.py b/test/test_remove_unused_imports.py new file mode 100644 index 00000000..9ba6a035 --- /dev/null +++ b/test/test_remove_unused_imports.py @@ -0,0 +1,295 @@ +import ast + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unused_imports(source): + module = ast.parse(source, 'remove_unused_imports') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=True, remove_variables=False, remove_definitions=False, remove_unreachable=False) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unused_imports(source) + compare_ast(expected_ast, actual_ast) + + +def test_unused_import(): + run_test(''' +import os +print(1) +''', 'print(1)') + + +def test_used_import(): + source = ''' +import os +print(os.getcwd()) +''' + run_test(source, source) + + +def test_unused_names_in_import(): + run_test(''' +import os, sys, json +print(sys.argv) +''', ''' +import sys +print(sys.argv) +''') + + +def test_unused_names_in_from_import(): + run_test(''' +from os.path import join, exists, isdir +print(join('a', 'b'), exists('c')) +''', ''' +from os.path import join, exists +print(join('a', 'b'), exists('c')) +''') + + +def test_unused_from_import_with_relative_module(): + run_test(''' +from . import sibling +from .. import parent +print(parent) +''', ''' +from .. import parent +print(parent) +''') + + +def test_unused_alias(): + run_test(''' +import numpy as np +from os import path as p +print(1) +''', 'print(1)') + + +def test_used_alias_hides_original_name(): + source = ''' +import numpy as np +from os import path as p +print(np, p) +''' + run_test(source, source) + + +def test_original_name_of_alias_is_not_a_use(): + run_test(''' +import numpy as np +print(numpy) +''', 'print(numpy)') + + +def test_dotted_import_binds_root_package(): + run_test(''' +import os.path +print(1) +''', 'print(1)') + + source = ''' +import os.path +print(os.sep) +''' + run_test(source, source) + + +def test_dotted_imports_of_same_package_are_used_together(): + source = ''' +import xml.dom +import xml.etree +print(xml.dom) +''' + run_test(source, source) + + +def test_future_import_is_kept(): + source = ''' +from __future__ import division +print(1) +''' + run_test(source, source) + + +def test_import_used_in_function(): + source = ''' +import os +def f(): + return os.getcwd() +''' + run_test(source, source) + + +def test_import_used_in_class_body(): + source = ''' +import os +class C: + path = os.getcwd() +''' + run_test(source, source) + + +def test_import_used_in_decorator_and_default(): + source = ''' +import functools +import sys +@functools.lru_cache +def f(a=sys.maxsize): + pass +''' + run_test(source, source) + + +def test_unused_import_in_function(): + run_test(''' +def f(): + import os + return 1 +''', ''' +def f(): + return 1 +''') + + +def test_function_import_is_not_used_by_other_scopes(): + run_test(''' +import os +def f(): + import os + return os.getcwd() +''', ''' +def f(): + import os + return os.getcwd() +''') + + +def test_import_in_class_body_is_kept(): + source = ''' +class C: + import os +''' + run_test(source, source) + + +def test_empty_suite_is_replaced(): + run_test(''' +def f(): + import os +''', ''' +def f(): + 0 +''') + + run_test(''' +try: + import ujson +except ImportError: + pass +''', ''' +try: + 0 +except ImportError: + pass +''') + + +def test_module_with_only_unused_imports_is_empty(): + run_test(''' +import os +from sys import argv +''', '') + + +def test_import_names_listed_in_all_are_kept(): + source = ''' +import os +import sys +__all__ = ['os'] +''' + run_test(source, ''' +import os +__all__ = ['os'] +''') + + +def test_import_names_added_to_all_are_kept(): + run_test(''' +import os +import sys +import json +__all__ = ['os'] +__all__ += ['sys'] +''', ''' +import os +import sys +__all__ = ['os'] +__all__ += ['sys'] +''') + + +def test_dynamic_all_keeps_every_import(): + source = ''' +import os +__all__ = sorted(['os']) +''' + run_test(source, source) + + source = ''' +import os +__all__ = [] +__all__.append('os') +''' + run_test(source, source) + + +def test_star_import_taints_module(): + source = ''' +from os import * +import sys +''' + run_test(source, source) + + +def test_exec_taints_module(): + source = ''' +import sys +exec('print(sys)') +''' + run_test(source, source) + + +def test_name_assigned_after_import_is_not_a_use(): + run_test(''' +import os +os = None +''', ''' +os = None +''') + + +def test_import_used_by_augmented_assignment_is_kept(): + source = ''' +import os +os += 1 +''' + run_test(source, source) + + +def test_global_declaration_is_a_use(): + source = ''' +import os +def f(): + global os + os = None +''' + run_test(source, source) diff --git a/test/test_remove_unused_variables.py b/test/test_remove_unused_variables.py new file mode 100644 index 00000000..13d503a4 --- /dev/null +++ b/test/test_remove_unused_variables.py @@ -0,0 +1,364 @@ +import ast +import sys + +import pytest + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unused_variables(source): + module = ast.parse(source, 'remove_unused_variables') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=False, remove_variables=True, remove_definitions=False, remove_unreachable=False) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unused_variables(source) + compare_ast(expected_ast, actual_ast) + + +def test_unused_variable(): + run_test(''' +x = 42 +print(1) +''', 'print(1)') + + +def test_used_variable(): + source = ''' +x = 42 +print(x) +''' + run_test(source, source) + + +def test_side_effect_free_values(): + run_test(''' +a = 'text' +b = [1, 2, {3: (4, 5)}] +c = 1 + 2 * 3 +d = not a +e = 1 if a else 2 +f = a.attribute +g = a[1:2] +h = lambda: 1 +i = len +j = a and b or c +k = a < b +print(1) +''', 'print(1)') + + +def test_values_with_side_effects_are_kept(): + source = ''' +a = call() +b = [call()] +c = {1: call()} +d = 1 + call() +e = [i for i in range(3)] +f = a.method() +g = lambda x=call(): x +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason='unpacking in displays needs python 3.5') +def test_keeps_value_that_unpacks(): + source = ''' +a = [*b] +c = {**d} +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason='f-strings and annotated assignments need python 3.6') +def test_f_string_value(): + run_test(''' +a = f'{b}' +c = f'{call()}' +print(1) +''', ''' +c = f'{call()}' +print(1) +''') + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason='async functions need python 3.5') +def test_await_and_yield_values_are_kept(): + source = ''' +async def f(): + x = await call() +def g(): + y = yield 1 +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason='annotated assignments need python 3.6') +def test_annotated_assignment(): + run_test(''' +x: int = 42 +y: int +z: int = call() +print(1) +''', ''' +y: int +z: int = call() +print(1) +''') + + source = ''' +x: call() = 1 +''' + run_test(source, source) + + +def test_chained_and_unpacking_targets_are_kept(): + source = ''' +a = b = 1 +c, d = 1, 2 +[e, f] = [1, 2] +''' + run_test(source, source) + + +def test_attribute_and_subscript_targets_are_kept(): + source = ''' +obj.x = 1 +arr[0] = 99 +''' + run_test(source, source) + + +def test_augmented_assignment_is_a_use(): + source = ''' +x = 0 +x += 1 +''' + run_test(source, source) + + +def test_del_is_a_use(): + source = ''' +x = 0 +del x +''' + run_test(source, source) + + +def test_loop_and_with_targets_are_kept(): + source = ''' +for i in range(3): + pass +with open('f') as fp: + pass +''' + run_test(source, source) + + +def test_dunder_names_are_kept(): + source = ''' +__version__ = '1.0' +__author__ = 'me' +''' + run_test(source, source) + + +def test_names_listed_in_all_are_kept(): + run_test(''' +__all__ = ['x'] +x = 1 +y = 2 +''', ''' +__all__ = ['x'] +x = 1 +''') + + +def test_dynamic_all_keeps_every_module_variable(): + source = ''' +__all__ = list_names() +x = 1 +''' + run_test(source, source) + + run_test(''' +__all__ = list_names() +def f(): + y = 1 +''', ''' +__all__ = list_names() +def f(): + 0 +''') + + +def test_variable_used_in_nested_function(): + source = ''' +x = 1 +def f(): + return x +''' + run_test(source, source) + + +def test_unused_variable_in_function(): + run_test(''' +def f(): + x = 1 + return 2 +''', ''' +def f(): + return 2 +''') + + +def test_function_variable_is_not_used_by_other_scopes(): + run_test(''' +x = 1 +def f(): + x = 2 + return 3 +print(x) +''', ''' +x = 1 +def f(): + return 3 +print(x) +''') + + run_test(''' +x = 1 +def f(): + x = 2 + return 3 +''', ''' +def f(): + return 3 +''') + + +def test_variable_used_by_inner_function(): + source = ''' +def f(): + x = 1 + def g(): + return x + return g +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='nonlocal needs python 3') +def test_nonlocal_declaration_is_a_use(): + source = ''' +def f(): + x = 1 + def g(): + nonlocal x + x = 2 + return g +''' + run_test(source, source) + + +def test_global_declaration_is_a_use(): + source = ''' +x = 1 +def f(): + global x + x = 2 +''' + run_test(source, source) + + +def test_class_attributes_are_kept(): + source = ''' +class C: + x = 1 +''' + run_test(source, source) + + +def test_variable_in_class_method(): + run_test(''' +class C: + def method(self): + x = 1 + return 2 +''', ''' +class C: + def method(self): + return 2 +''') + + +def test_variable_in_class_in_function_is_kept(): + source = ''' +def f(): + class C: + x = 1 + return C +''' + run_test(source, source) + + +def test_empty_suite_is_replaced(): + run_test(''' +def f(): + x = 1 +''', ''' +def f(): + 0 +''') + + run_test(''' +if a: + x = 1 +else: + y = 2 +''', ''' +if a: + 0 +else: + 0 +''') + + +def test_module_with_only_unused_variables_is_empty(): + run_test(''' +a = 1 +b = 2 +''', '') + + +def test_removing_a_variable_can_make_another_unused(): + run_test(''' +a = 1 +b = a +c = b + 1 +print(1) +''', 'print(1)') + + +def test_locals_taints_module(): + source = ''' +def f(): + x = 1 + return locals() +''' + run_test(source, source) + + +def test_star_import_taints_module(): + source = ''' +from os import * +x = 1 +''' + run_test(source, source)