diff --git a/CHANGELOG.md b/CHANGELOG.md index ee1c1792..a68aef8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ and will output source code compatible with the version of the interpreter it is This means that if you minify code written for Python 3.11 using python-minifier running with Python 3.12, the minified code may only run with Python 3.12. +## [Unreleased] + +### Added +- New transforms to remove dead code. They are all disabled by default, and repeat until there is nothing left to remove: + + Remove unused imports, enabled with the `--remove-unused-imports` option. + + Remove assignments to unused variables, enabled with the `--remove-unused-variables` option. + + Remove unused function and class definitions, enabled with the `--remove-unused-definitions` option. + + Remove statements that follow a `return`, `raise`, `break` or `continue`, enabled with the `--remove-unreachable` option. + ## [3.4.0] - 2026-09-29 ### Added diff --git a/docs/source/minification_options/index.rst b/docs/source/minification_options/index.rst index e19b2eab..0e2d8457 100644 --- a/docs/source/minification_options/index.rst +++ b/docs/source/minification_options/index.rst @@ -31,4 +31,8 @@ They can be enabled or disabled through the minify function, or passing options rename_globals remove_asserts remove_debug + remove_unused_imports + remove_unused_variables + remove_unused_definitions + remove_unreachable prefer_single_line diff --git a/docs/source/minification_options/remove_unreachable.py b/docs/source/minification_options/remove_unreachable.py new file mode 100644 index 00000000..08ad2301 --- /dev/null +++ b/docs/source/minification_options/remove_unreachable.py @@ -0,0 +1,24 @@ +def check(value): + if value < 0: + raise ValueError(value) + print('never runs') + + if value: + return 'set' + else: + return 'unset' + + print('never runs') + + +def first(items): + for item in items: + if item: + return item + print('runs if there are no items') + return None + + +def generator(): + return + yield diff --git a/docs/source/minification_options/remove_unreachable.rst b/docs/source/minification_options/remove_unreachable.rst new file mode 100644 index 00000000..683e1bcc --- /dev/null +++ b/docs/source/minification_options/remove_unreachable.rst @@ -0,0 +1,38 @@ +Remove Unreachable +================== + +This transform removes statements that follow a statement that always leaves the block it is in. + +The statements after a ``return``, ``raise``, ``break`` or ``continue`` in the same block are removed, as they can never +run. The same is true for the statements after an ``if`` statement where every branch ends this way, so it needs an +``else`` clause. A ``try`` statement ends the block if its ``finally`` clause does, or if its body and all of its +``except`` clauses do. A ``with`` statement ends the block if its body does, unless it only does so by raising +an exception, as the context manager may suppress that. ``for`` and ``while`` loops never end the block, as their body +might not run. + +The statements are kept if removing them would change the meaning of the program, for example if they contain a ``yield``, +a ``global`` or ``nonlocal`` declaration, or the only assignment to a local variable that is used before it. + +If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used +in the module, nothing is removed. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. For example the names that were only used by unreachable statements may be removed +by :doc:`remove_unused_imports`. + +The transform is disabled by default. Enable it by passing the ``remove_unreachable=True`` argument to the :func:`python_minifier.minify` function, +or passing ``--remove-unreachable`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unreachable.py + +Output +~~~~~~ + +.. literalinclude:: remove_unreachable.min.py + :language: python diff --git a/docs/source/minification_options/remove_unused_definitions.py b/docs/source/minification_options/remove_unused_definitions.py new file mode 100644 index 00000000..61076c14 --- /dev/null +++ b/docs/source/minification_options/remove_unused_definitions.py @@ -0,0 +1,25 @@ +def helper(): + return 1 + + +def unused(): + return helper() + + +class Unused: + def method(self): + return 1 + + +@register +def registered(): + pass + + +def main(): + def inner(): + pass + return 2 + + +print(main()) diff --git a/docs/source/minification_options/remove_unused_definitions.rst b/docs/source/minification_options/remove_unused_definitions.rst new file mode 100644 index 00000000..e7a47633 --- /dev/null +++ b/docs/source/minification_options/remove_unused_definitions.rst @@ -0,0 +1,47 @@ +Remove Unused Definitions +========================= + +This transform removes functions and classes that are never used. + +A function or class definition is removed if its name is not used anywhere in the module. Methods and the other names +defined in a class body are never removed, only functions and classes defined at module level or inside a function. + +A definition is kept if executing it could have an effect, which is when: + + - It has a decorator, as the decorator may register the function or class somewhere + - A default value, annotation, base class or keyword evaluates something that isn't known to be harmless, like a call + - The body of a class runs anything that isn't a simple assignment, a docstring or a definition + +This is not always safe, so the transform is disabled by default. It could break any program that imports a +definition from the minified module, or looks up definitions by a string name. A class without a decorator could +still be registered by a base class or metaclass. + +Definitions are never removed: + + - If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used + in the module + - If the name is a dunder name like ``__getattr__`` + - If the name is included as a literal string in ``__all__`` at module level, or ``__all__`` is used in a way that + can't be determined by looking at the module + +If a definition is removed and a statement is still required, it is replaced by a zero expression statement. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. For example removing a function also removes any helper that only that function used. + +Enable this source transformation by passing the ``remove_unused_definitions=True`` argument to the :func:`python_minifier.minify` +function, or passing ``--remove-unused-definitions`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unused_definitions.py + +Output +~~~~~~ + +.. literalinclude:: remove_unused_definitions.min.py + :language: python diff --git a/docs/source/minification_options/remove_unused_imports.py b/docs/source/minification_options/remove_unused_imports.py new file mode 100644 index 00000000..ece44705 --- /dev/null +++ b/docs/source/minification_options/remove_unused_imports.py @@ -0,0 +1,12 @@ +import os +import sys, json +from collections import OrderedDict, defaultdict +from os import path as os_path + +print(sys.argv) +print(defaultdict(list)) + + +def listing(): + import glob + return [] diff --git a/docs/source/minification_options/remove_unused_imports.rst b/docs/source/minification_options/remove_unused_imports.rst new file mode 100644 index 00000000..802ca854 --- /dev/null +++ b/docs/source/minification_options/remove_unused_imports.rst @@ -0,0 +1,46 @@ +Remove Unused Imports +===================== + +This transform removes imports of names that are never used. + +An import statement is removed if none of the names it binds are used. If only some of the names are unused, just those +names are removed from the statement. A name is used if it is read anywhere in the module, including in nested +functions, decorators and default arguments. + +This is not always safe, so the transform is disabled by default. It could break: + + - Any program that imports a name from the minified module, as nothing is known about how the module is used elsewhere + - Imports that are only used for their side effects, like registering a codec + - Names that are only referred to by strings, for example type annotations that are string literals or code run by ``getattr()`` + +Imports are never removed: + + - If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used + in the module + - If they are ``__future__`` imports + - If the name is included as a literal string in ``__all__`` at module level, or ``__all__`` is used in a way that + can't be determined by looking at the module + - If they are in a class body, where the names become attributes of the class + +If an import statement is removed and a statement is still required, it is replaced by a zero expression statement. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. For example if :doc:`remove_unused_definitions` removes a function, the imports that +were only used by that function are removed too. + +Enable this source transformation by passing the ``remove_unused_imports=True`` argument to the :func:`python_minifier.minify` +function, or passing ``--remove-unused-imports`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unused_imports.py + +Output +~~~~~~ + +.. literalinclude:: remove_unused_imports.min.py + :language: python diff --git a/docs/source/minification_options/remove_unused_variables.py b/docs/source/minification_options/remove_unused_variables.py new file mode 100644 index 00000000..ad409a34 --- /dev/null +++ b/docs/source/minification_options/remove_unused_variables.py @@ -0,0 +1,18 @@ +TIMEOUT = 30 +RETRIES = TIMEOUT * 2 +VERSION = '1.0' +__version__ = VERSION + +started = start() + + +def process(items): + total = 0 + scratch = [item for item in items] + lookup = {'a': 1} + for item in items: + total += item + return total + + +print(RETRIES) diff --git a/docs/source/minification_options/remove_unused_variables.rst b/docs/source/minification_options/remove_unused_variables.rst new file mode 100644 index 00000000..45ffa3fb --- /dev/null +++ b/docs/source/minification_options/remove_unused_variables.rst @@ -0,0 +1,48 @@ +Remove Unused Variables +======================= + +This transform removes assignments to variables that are never used. + +An assignment is removed if the name is not used anywhere in the module, and evaluating the assigned value is known not +to have any effect. Assigning a literal, a collection of literals, a name, an attribute, a subscript, a lambda, +or an expression made of operators with these is removed. Anything that calls a function, uses ``await`` or ``yield``, +or is a comprehension is kept, as is any value that might unpack an iterable or mapping. + +Only assignments that bind a single name are removed, including annotated assignments with a value. +Assignments to attributes and subscripts, to multiple targets, and augmented assignments are always kept. +Using a name with ``del``, an augmented assignment or ``global`` and ``nonlocal`` declarations all count as a use. + +This is not always safe, so the transform is disabled by default. It could break any program that imports a +variable from the minified module, or reads variables by a string name. Reading an attribute or subscript, and applying +an operator, are assumed to have no effect, but they may run code that does. + +Assignments are never removed: + + - If ``eval()``, ``exec()``, ``locals()``, ``globals()``, ``vars()`` are used, or ``from import *`` is used + in the module + - If the name is a dunder name like ``__version__`` + - If the name is included as a literal string in ``__all__`` at module level, or ``__all__`` is used in a way that + can't be determined by looking at the module + - If they are in a class body, where the names become attributes of the class + +If an assignment is removed and a statement is still required, it is replaced by a zero expression statement. + +Removing code can leave other code unused, so this transform is run again with the other dead code transforms until +there is nothing left to remove. + +Enable this source transformation by passing the ``remove_unused_variables=True`` argument to the :func:`python_minifier.minify` +function, or passing ``--remove-unused-variables`` to the pyminify command. + +Example +------- + +Input +~~~~~ + +.. literalinclude:: remove_unused_variables.py + +Output +~~~~~~ + +.. literalinclude:: remove_unused_variables.min.py + :language: python diff --git a/src/python_minifier/__init__.py b/src/python_minifier/__init__.py index 8b091d2b..88aed9ea 100644 --- a/src/python_minifier/__init__.py +++ b/src/python_minifier/__init__.py @@ -22,6 +22,7 @@ ) from python_minifier.transforms.combine_imports import CombineImports from python_minifier.transforms.constant_folding import FoldConstants +from python_minifier.transforms.dead_code import eliminate_dead_code from python_minifier.transforms.remove_annotations import RemoveAnnotations from python_minifier.transforms.remove_annotations_options import RemoveAnnotationsOptions from python_minifier.transforms.remove_asserts import RemoveAsserts @@ -75,7 +76,11 @@ def minify( remove_builtin_exception_brackets=True, constant_folding=True, prefer_single_line=False, - remove_dead_branches=True + remove_dead_branches=True, + remove_unused_imports=False, + remove_unused_variables=False, + remove_unused_definitions=False, + remove_unreachable=False ): """ Minify a python module @@ -112,6 +117,10 @@ def minify( :param bool constant_folding: If literal expressions should be evaluated :param bool prefer_single_line: If semi-colons should be preferred over newlines where there is no difference in output size :param bool remove_dead_branches: If if-statements with a constant False test should be removed + :param bool remove_unused_imports: If imported names that are never used should be removed + :param bool remove_unused_variables: If assignments to variables that are never used should be removed + :param bool remove_unused_definitions: If functions and classes that are never used should be removed + :param bool remove_unreachable: If statements that follow a return, raise, break or continue should be removed :rtype: str @@ -164,6 +173,15 @@ def minify( if remove_dead_branches: module = RemoveDeadBranches()(module) + if remove_unused_imports or remove_unused_variables or remove_unused_definitions or remove_unreachable: + module = eliminate_dead_code( + module, + remove_imports=remove_unused_imports, + remove_variables=remove_unused_variables, + remove_definitions=remove_unused_definitions, + remove_unreachable=remove_unreachable + ) + if remove_explicit_return_none: module = RemoveExplicitReturnNone()(module) diff --git a/src/python_minifier/__init__.pyi b/src/python_minifier/__init__.pyi index fde0d748..50c596f9 100644 --- a/src/python_minifier/__init__.pyi +++ b/src/python_minifier/__init__.pyi @@ -30,7 +30,11 @@ def minify( remove_builtin_exception_brackets: bool = ..., constant_folding: bool = ..., prefer_single_line: bool = ..., - remove_dead_branches: bool = ... + remove_dead_branches: bool = ..., + remove_unused_imports: bool = ..., + remove_unused_variables: bool = ..., + remove_unused_definitions: bool = ..., + remove_unreachable: bool = ... ) -> Text: ... diff --git a/src/python_minifier/__main__.py b/src/python_minifier/__main__.py index bd524047..79f582b3 100644 --- a/src/python_minifier/__main__.py +++ b/src/python_minifier/__main__.py @@ -255,6 +255,30 @@ def parse_args(): help='Disable removing branches that are never executed', dest='remove_dead_branches', ) + minification_options.add_argument( + '--remove-unused-imports', + action='store_true', + help='Enable removing imports of names that are never used', + dest='remove_unused_imports', + ) + minification_options.add_argument( + '--remove-unused-variables', + action='store_true', + help='Enable removing assignments to variables that are never used', + dest='remove_unused_variables', + ) + minification_options.add_argument( + '--remove-unused-definitions', + action='store_true', + help='Enable removing functions and classes that are never used', + dest='remove_unused_definitions', + ) + minification_options.add_argument( + '--remove-unreachable', + action='store_true', + help='Enable removing statements that follow a return, raise, break or continue', + dest='remove_unreachable', + ) annotation_options = parser.add_argument_group('remove annotations options', 'Options that affect how annotations are removed') annotation_options.add_argument( @@ -391,6 +415,10 @@ def do_minify(source, filename, minification_args): remove_builtin_exception_brackets=minification_args.remove_exception_brackets, constant_folding=minification_args.constant_folding, prefer_single_line=minification_args.prefer_single_line, + remove_unused_imports=minification_args.remove_unused_imports, + remove_unused_variables=minification_args.remove_unused_variables, + remove_unused_definitions=minification_args.remove_unused_definitions, + remove_unreachable=minification_args.remove_unreachable, ) # Encode minified result to bytes for comparison and output diff --git a/src/python_minifier/transforms/dead_code.py b/src/python_minifier/transforms/dead_code.py new file mode 100644 index 00000000..04bd9257 --- /dev/null +++ b/src/python_minifier/transforms/dead_code.py @@ -0,0 +1,54 @@ +from python_minifier.rename import add_namespace, bind_names, resolve_names +from python_minifier.transforms.remove_unreachable import RemoveUnreachable +from python_minifier.transforms.remove_unused_definitions import RemoveUnusedDefinitions +from python_minifier.transforms.remove_unused_imports import RemoveUnusedImports +from python_minifier.transforms.remove_unused_variables import RemoveUnusedVariables + + +def eliminate_dead_code(module, remove_imports, remove_variables, remove_definitions, remove_unreachable): + """ + Remove code that has no effect from a module + + Removing code can make more code dead, for example removing an unused function can leave the names + it used unused. This repeats until there is nothing left to remove. + + Modules that are tainted, because they use features like star imports or ``exec``, are not changed. + + The names in the module are left unbound on return. + + :param module: The module to remove dead code from + :type module: ast.Module + :param bool remove_imports: If unused imports should be removed + :param bool remove_variables: If assignments to unused variables should be removed + :param bool remove_definitions: If unused function and class definitions should be removed + :param bool remove_unreachable: If statements after return, raise, break or continue should be removed + :rtype: ast.Module + """ + + transforms = [] + if remove_unreachable: + transforms.append(RemoveUnreachable()) + if remove_definitions: + transforms.append(RemoveUnusedDefinitions()) + if remove_variables: + transforms.append(RemoveUnusedVariables()) + if remove_imports: + transforms.append(RemoveUnusedImports()) + + changed = True + while changed: + # Names must be bound again to account for what was removed in the last pass + add_namespace(module) + bind_names(module) + resolve_names(module) + + if module.tainted: + break + + changed = False + for transform in transforms: + module = transform(module) + changed = changed or transform.changed + + add_namespace(module) + return module diff --git a/src/python_minifier/transforms/remove_unreachable.py b/src/python_minifier/transforms/remove_unreachable.py new file mode 100644 index 00000000..bab5dbe0 --- /dev/null +++ b/src/python_minifier/transforms/remove_unreachable.py @@ -0,0 +1,116 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.remove_dead_branches import in_function_scope +from python_minifier.transforms.scope_analysis import DeadCodeTransformer, makes_local + +_TRY = tuple(getattr(ast, name) for name in ('Try', 'TryExcept', 'TryFinally') if hasattr(ast, name)) + + +def _suite_terminates(suite, raises): + return bool(suite) and _terminates(suite[-1], raises) + + +def _terminates(node, raises=True): + """ + Does control never continue to the statement after this one? + + :param node: The statement to check + :type node: ast.AST + :param bool raises: If a ``raise`` statement counts as terminating + :rtype: bool + """ + + if isinstance(node, (ast.Return, ast.Break, ast.Continue)): + return True + + if isinstance(node, ast.Raise): + return raises + + if isinstance(node, ast.If): + return bool(node.orelse) and _suite_terminates(node.body, raises) and _suite_terminates(node.orelse, raises) + + if isinstance(node, ast.TryStar): + return _suite_terminates(node.finalbody, raises) + + if isinstance(node, _TRY): + if _suite_terminates(getattr(node, 'finalbody', []), raises): + return True + + if not (_suite_terminates(node.body, raises) or _suite_terminates(getattr(node, 'orelse', []), raises)): + return False + + return all(_suite_terminates(handler.body, raises) for handler in getattr(node, 'handlers', [])) + + if isinstance(node, (ast.With, ast.AsyncWith)): + # The context manager can suppress an exception raised in the body + return _suite_terminates(node.body, raises=False) + + # Loops may never run their body, and the else clause of a loop runs when it is not broken out of + return False + + +class RemoveUnreachable(DeadCodeTransformer): + """ + Remove statements that follow a statement that always return, raises, breaks or continues + + Statements are kept if removing them would change the meaning of the program, for example if + they contain a ``yield`` or the only assignment to a local variable that is used earlier. + """ + + def suite(self, node_list, parent): + reachable = [] + + for index, node in enumerate(node_list): + reachable.append(self.visit(node)) + + unreachable = node_list[index + 1:] + if unreachable and _terminates(node) and self.can_remove(unreachable): + self.changed = True + break + + return reachable + + def can_remove(self, statements): + """ + Can these statements be removed without changing the meaning of the enclosing scope? + + :param list statements: Consecutive statements in a suite + :rtype: bool + """ + + namespace = statements[0].namespace + nodes = [node for statement in statements for node in ast.walk(statement)] + + for node in nodes: + if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Global, ast.Nonlocal)) and getattr(node, 'namespace', None) is namespace: + return False + + if isinstance(namespace, ast.Module) or (isinstance(namespace, ast.ClassDef) and not in_function_scope(namespace)): + # Name resolution is dynamic in these scopes + return True + + removed = {id(node) for node in nodes} + local_bindings = {id(binding) for binding in namespace.bindings} + + for node in nodes: + if getattr(node, 'namespace', None) is not namespace or not makes_local(node): + continue + + binding = self.scope.binding(node) + if binding is None: + continue + + if isinstance(namespace, ast.ClassDef): + # A class body binding hides the enclosing function's name from loads in the class body only + references = [reference for reference in binding.references if getattr(reference, 'namespace', None) is namespace] + elif id(binding) in local_bindings: + references = binding.references + else: + continue + + # Without these statements the name would no longer be local to this namespace + remaining = [reference for reference in references if id(reference) not in removed] + if remaining and not any(makes_local(reference) for reference in remaining): + return False + + return True diff --git a/src/python_minifier/transforms/remove_unused_definitions.py b/src/python_minifier/transforms/remove_unused_definitions.py new file mode 100644 index 00000000..71ee19dd --- /dev/null +++ b/src/python_minifier/transforms/remove_unused_definitions.py @@ -0,0 +1,23 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.scope_analysis import DeadCodeTransformer +from python_minifier.transforms.side_effects import is_definition_side_effect_free + + +class RemoveUnusedDefinitions(DeadCodeTransformer): + """ + Remove function and class definitions that are never used + + Decorated definitions are never removed, as the decorator may register the definition somewhere. + Neither are definitions that evaluate something with side effects when they are executed. + """ + + def eliminate(self, node): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return node + + if not is_definition_side_effect_free(node) or not self.scope.is_removable(node, node.name): + return node + + self.changed = True + return None diff --git a/src/python_minifier/transforms/remove_unused_imports.py b/src/python_minifier/transforms/remove_unused_imports.py new file mode 100644 index 00000000..f9e8f423 --- /dev/null +++ b/src/python_minifier/transforms/remove_unused_imports.py @@ -0,0 +1,42 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.scope_analysis import DeadCodeTransformer + + +class RemoveUnusedImports(DeadCodeTransformer): + """ + Remove imported names that are never used + + An import statement is removed when none of the names it binds are used. If only some of them are unused, + only those names are removed from the statement. + + ``__future__`` imports are never removed. + """ + + def eliminate(self, node): + if not isinstance(node, (ast.Import, ast.ImportFrom)): + return node + + if isinstance(node, ast.ImportFrom) and node.module == '__future__': + return node + + used = [alias for alias in node.names if not self.is_unused(alias)] + if len(used) == len(node.names): + return node + + self.changed = True + + if not used: + return None + + node.names = used + return node + + def is_unused(self, alias): + if alias.name == '*': + return False + + # A dotted import without an alias binds only the root package + name = alias.asname or alias.name.split('.')[0] + + return self.scope.is_removable(alias, name) diff --git a/src/python_minifier/transforms/remove_unused_variables.py b/src/python_minifier/transforms/remove_unused_variables.py new file mode 100644 index 00000000..e4a969ed --- /dev/null +++ b/src/python_minifier/transforms/remove_unused_variables.py @@ -0,0 +1,34 @@ +import python_minifier.ast_compat as ast + +from python_minifier.transforms.scope_analysis import DeadCodeTransformer +from python_minifier.transforms.side_effects import is_side_effect_free + + +class RemoveUnusedVariables(DeadCodeTransformer): + """ + Remove assignments to variables that are never used + + Only assignments of a single name are removed, and only when the assigned value is known to have no side effects. + """ + + def eliminate(self, node): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + annotation = None + elif isinstance(node, ast.AnnAssign) and node.value is not None: + target = node.target + annotation = node.annotation + else: + return node + + if not isinstance(target, ast.Name): + return node + + if not is_side_effect_free(node.value) or not is_side_effect_free(annotation): + return node + + if not self.scope.is_removable(target, target.id): + return node + + self.changed = True + return None diff --git a/src/python_minifier/transforms/scope_analysis.py b/src/python_minifier/transforms/scope_analysis.py new file mode 100644 index 00000000..b5ddb8a6 --- /dev/null +++ b/src/python_minifier/transforms/scope_analysis.py @@ -0,0 +1,209 @@ +import python_minifier.ast_compat as ast +from python_minifier.ast_annotation import get_parent +from python_minifier.transforms.suite_transformer import SuiteTransformer +from python_minifier.util import is_constant_node + + +def is_dunder_name(name): + """ + Is this a name like __init__, which has a special meaning to python + + :param str name: The name to check + :rtype: bool + """ + + return name.startswith('__') and name.endswith('__') + + +def reads_name(reference): + """ + Does a reference to a binding use the current value of the name? + + A binding that is only referenced by the statements that define it is never used. + Anything else that mentions the name counts as a use, including ``del`` statements, + augmented assignments and ``global`` or ``nonlocal`` declarations, which let other + scopes read and write the name. + + :param reference: A node from the references of a binding + :type reference: ast.AST + :rtype: bool + """ + + if isinstance(reference, ast.Name): + if isinstance(reference.ctx, (ast.Load, ast.Del)): + return True + + if isinstance(reference.ctx, ast.Store): + try: + return isinstance(get_parent(reference), ast.AugAssign) + except ValueError: + return True + + return False + + return isinstance(reference, (ast.Global, ast.Nonlocal)) + + +def makes_local(reference): + """ + Does a reference to a binding cause the name to be local to its namespace? + + :param reference: A node from the references of a binding + :type reference: ast.AST + :rtype: bool + """ + + if isinstance(reference, ast.Name): + return not isinstance(reference.ctx, ast.Load) + + return not isinstance(reference, (ast.Global, ast.Nonlocal)) + + +def find_exports(module): + """ + Find the names a module lists in ``__all__`` + + :param module: The module to search + :type module: ast.Module + :return: The listed names, or None if ``__all__`` is used in a way that can't be determined statically + :rtype: set[str] or None + """ + + names = set() + + for node in ast.walk(module): + if isinstance(node, ast.Name) and node.id == '__all__' and isinstance(node.ctx, ast.Load): + return None + + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): + targets = [node.target] + else: + continue + + if not any(isinstance(target, ast.Name) and target.id == '__all__' for target in targets): + continue + + if node.value is None: + continue + + if not isinstance(node.value, (ast.List, ast.Tuple, ast.Set)): + return None + + for element in node.value.elts: + if not is_constant_node(element, ast.Str): + return None + names.add(element.s) + + return names + + +class ScopeAnalysis(object): + """ + Answers questions about how names are used in a module + + The module must have had its names bound and resolved, the analysis is invalid once + the module is changed. + + :param module: The module to analyse + :type module: ast.Module + """ + + def __init__(self, module): + self._binding_for_reference = {} + + for node in ast.walk(module): + for binding in getattr(node, 'bindings', ()): + for reference in binding.references: + self._binding_for_reference[id(reference)] = binding + + self._exports = find_exports(module) + + def binding(self, node): + """ + The binding that a node creates or references + + :type node: ast.AST + :rtype: python_minifier.rename.binding.NameBinding or None + """ + + return self._binding_for_reference.get(id(node)) + + def is_removable(self, node, name): + """ + Can the statement that makes a node bind a name be removed, as far as the name is concerned? + + The name must never be used in the module. Names that could be used from outside the + module, or that the interpreter gives a meaning to, are never unused. Class namespaces + are skipped as the names in them become attributes of the class. + + :param node: The node that binds the name. This is a Name, alias, FunctionDef, AsyncFunctionDef or ClassDef + :type node: ast.AST + :param str name: The name being bound + :rtype: bool + """ + + namespace = node.namespace + + if isinstance(namespace, ast.ClassDef) or is_dunder_name(name): + return False + + if isinstance(namespace, ast.Module) and (self._exports is None or name in self._exports): + return False + + binding = self.binding(node) + if binding is None: + return False + + return not any(reads_name(reference) for reference in binding.references) + + +class DeadCodeTransformer(SuiteTransformer): + """ + Base class for transforms that remove statements that have no effect on a bound module + + The :attr:`changed` attribute is set if the last run removed anything. + """ + + def __call__(self, module): + self.changed = False + self.scope = ScopeAnalysis(module) + return self.visit(module) + + def eliminate(self, node): + """ + Remove a statement, or part of it + + :param node: The statement to consider + :type node: ast.AST + :return: The statement to keep, or None if it should be removed + :rtype: ast.AST or None + """ + + return node + + def suite(self, node_list, parent): + without_dead_code = [] + + for node in node_list: + node = self.eliminate(node) + if node is not None: + without_dead_code.append(self.visit(node)) + + return self.non_empty(without_dead_code, parent) + + def non_empty(self, suite, parent): + """ + Ensure a suite has a statement, if it needs one + + :param list suite: The statements in the suite + :param parent: The node that owns the suite + :type parent: ast.AST + :rtype: list + """ + + if suite or isinstance(parent, ast.Module): + return suite + + return [self.add_child(ast.Expr(value=ast.Num(0)), parent=parent)] diff --git a/src/python_minifier/transforms/side_effects.py b/src/python_minifier/transforms/side_effects.py new file mode 100644 index 00000000..b514828b --- /dev/null +++ b/src/python_minifier/transforms/side_effects.py @@ -0,0 +1,129 @@ +import python_minifier.ast_compat as ast +from python_minifier.util import is_constant_node + +_FSTRINGS = tuple(getattr(ast, name) for name in ('JoinedStr', 'FormattedValue') if hasattr(ast, name)) + + +def is_side_effect_free(node): + """ + Is evaluating an expression, and discarding the result, known to do nothing? + + The analysis is conservative, anything that is not known to be harmless is assumed to + have side effects. Calls, ``await``, ``yield``, assignment expressions and + comprehensions are never side effect free. Looking up a name, accessing an + attribute or subscript and applying an operator are assumed not to run code that + matters, and so are not considered to have side effects. + + :param node: The expression to check, or None + :type node: ast.AST or None + :rtype: bool + """ + + if node is None: + return True + + if is_constant_node(node, (ast.Num, ast.Str, ast.Bytes, ast.NameConstant, ast.Ellipsis)) or isinstance(node, ast.Name): + return True + + if isinstance(node, (ast.Tuple, ast.List, ast.Set)): + return all(is_side_effect_free(element) for element in node.elts) + + if isinstance(node, ast.Dict): + return all(key is not None and is_side_effect_free(key) for key in node.keys) and all(is_side_effect_free(value) for value in node.values) + + if isinstance(node, ast.UnaryOp): + return is_side_effect_free(node.operand) + + if isinstance(node, ast.BinOp): + return is_side_effect_free(node.left) and is_side_effect_free(node.right) + + if isinstance(node, ast.BoolOp): + return all(is_side_effect_free(value) for value in node.values) + + if isinstance(node, ast.Compare): + return is_side_effect_free(node.left) and all(is_side_effect_free(comparator) for comparator in node.comparators) + + if isinstance(node, ast.IfExp): + return is_side_effect_free(node.test) and is_side_effect_free(node.body) and is_side_effect_free(node.orelse) + + if isinstance(node, ast.Attribute): + return is_side_effect_free(node.value) + + if isinstance(node, ast.Subscript): + return is_side_effect_free(node.value) and is_side_effect_free(node.slice) + + if isinstance(node, ast.Slice): + return is_side_effect_free(node.lower) and is_side_effect_free(node.upper) and is_side_effect_free(node.step) + + if isinstance(node, ast.Index): + return is_side_effect_free(node.value) + + if isinstance(node, ast.ExtSlice): + return all(is_side_effect_free(dim) for dim in node.dims) + + if isinstance(node, ast.Lambda): + return _are_defaults_side_effect_free(node.args) + + if isinstance(node, _FSTRINGS): + if hasattr(node, 'values'): + return all(is_side_effect_free(value) for value in node.values) + return is_side_effect_free(node.value) and is_side_effect_free(node.format_spec) + + return False + + +def _are_defaults_side_effect_free(arguments): + defaults = list(arguments.defaults) + list(getattr(arguments, 'kw_defaults', [])) + return all(is_side_effect_free(default) for default in defaults) + + +def is_definition_side_effect_free(node): + """ + Is executing a function or class definition, and discarding the result, known to do nothing? + + Defining a function evaluates its default values and annotations. Defining a class evaluates its + bases and keywords and runs its body. + Decorators are not considered, a decorated definition is never side effect free. + + :param node: The definition to check + :type node: ast.FunctionDef or ast.AsyncFunctionDef or ast.ClassDef + :rtype: bool + """ + + if node.decorator_list: + return False + + if isinstance(node, ast.ClassDef): + expressions = list(node.bases) + [keyword.value for keyword in getattr(node, 'keywords', [])] + expressions += [getattr(node, name) for name in ('starargs', 'kwargs') if getattr(node, name, None) is not None] + return all(is_side_effect_free(expression) for expression in expressions) and all(_is_class_statement_side_effect_free(statement) for statement in node.body) + + arguments = node.args + annotations = [getattr(node, 'returns', None)] + annotations += [getattr(arg, 'annotation', None) for arg in getattr(arguments, 'posonlyargs', []) + arguments.args + getattr(arguments, 'kwonlyargs', [])] + annotations += [getattr(arguments, name, None) for name in ('varargannotation', 'kwargannotation')] + if isinstance(arguments.vararg, ast.arg): + annotations.append(arguments.vararg.annotation) + if isinstance(arguments.kwarg, ast.arg): + annotations.append(arguments.kwarg.annotation) + + return _are_defaults_side_effect_free(arguments) and all(is_side_effect_free(annotation) for annotation in annotations) + + +def _is_class_statement_side_effect_free(statement): + if isinstance(statement, ast.Pass): + return True + + if isinstance(statement, ast.Expr): + return is_side_effect_free(statement.value) + + if isinstance(statement, ast.Assign): + return is_side_effect_free(statement.value) and all(isinstance(target, ast.Name) for target in statement.targets) + + if isinstance(statement, ast.AnnAssign): + return isinstance(statement.target, ast.Name) and is_side_effect_free(statement.annotation) and is_side_effect_free(statement.value) + + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return is_definition_side_effect_free(statement) + + return False diff --git a/test/test_dead_code.py b/test/test_dead_code.py new file mode 100644 index 00000000..c4def38b --- /dev/null +++ b/test/test_dead_code.py @@ -0,0 +1,209 @@ +import ast +import sys + +import pytest + +from python_minifier import minify +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code +from subprocess_compat import run_subprocess, safe_decode + +SOURCE = ''' +import os +import sys +import json + +unused_variable = 42 + +def unused_function(): + return json.dumps({}) + +class UnusedClass: + pass + +def main(): + unused_local = 1 + return sys.argv + print('unreachable') + +call(main()) +''' + + +def eliminate_all(source): + module = ast.parse(source, 'dead_code') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=True, remove_variables=True, remove_definitions=True, remove_unreachable=True) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = eliminate_all(source) + compare_ast(expected_ast, actual_ast) + + +def test_all_transforms_together(): + run_test(SOURCE, ''' +import sys + +def main(): + return sys.argv + +call(main()) +''') + + +def test_removing_a_function_makes_its_imports_unused(): + run_test(''' +import json +def helper(): + return json.dumps({}) +def main(): + return 1 +print(main()) +''', ''' +def main(): + return 1 +print(main()) +''') + + +def test_removing_unreachable_code_makes_names_unused(): + run_test(''' +import json +def helper(): + pass +def main(): + return 1 + return json.dumps(helper()) +print(main()) +''', ''' +def main(): + return 1 +print(main()) +''') + + +def test_removing_a_variable_makes_its_import_unused(): + run_test(''' +import json +value = json +print(1) +''', 'print(1)') + + +def test_values_with_side_effects_keep_their_imports(): + source = ''' +from os.path import join +path = join('a', 'b') +print(1) +''' + run_test(source, source) + + +def test_long_chain_of_dead_code(): + names = ['a%d' % i for i in range(20)] + source = '\n'.join('def %s():\n return %s()' % (name, following) for name, following in zip(names, names[1:])) + source += '\ndef %s():\n return 1\nprint(1)\n' % names[-1] + run_test(source, 'print(1)') + + +def test_nothing_to_remove(): + source = ''' +import sys +def main(): + return sys.argv +print(main()) +''' + run_test(source, source) + + +def test_tainted_module_is_not_changed(): + for source in ['from os import *\n' + SOURCE, SOURCE + "\neval('1')\n", 'import timeit\n' + SOURCE]: + expected_ast = ast.parse(source) + actual_ast = eliminate_all(source) + compare_ast(expected_ast, actual_ast) + + +def test_options_are_disabled_by_default(): + assert minify(SOURCE) == minify(SOURCE, remove_unused_imports=False, remove_unused_variables=False, remove_unused_definitions=False, remove_unreachable=False) + assert 'unused_function' in minify(SOURCE) + assert 'unused_variable' in minify(SOURCE) + assert 'UnusedClass' in minify(SOURCE) + assert 'unreachable' in minify(SOURCE) + assert 'import os' in minify(SOURCE) + + +def test_minify_options_are_independent(): + assert 'import os' not in minify(SOURCE, remove_unused_imports=True) + assert 'unused_function' in minify(SOURCE, remove_unused_imports=True) + + assert 'unused_variable' not in minify(SOURCE, remove_unused_variables=True) + assert 'import os' in minify(SOURCE, remove_unused_variables=True) + + assert 'unused_function' not in minify(SOURCE, remove_unused_definitions=True) + assert 'UnusedClass' not in minify(SOURCE, remove_unused_definitions=True) + assert 'unused_variable' in minify(SOURCE, remove_unused_definitions=True) + + assert 'unreachable' not in minify(SOURCE, remove_unreachable=True) + assert 'unused_function' in minify(SOURCE, remove_unreachable=True) + + +def test_minify_removes_everything_dead(): + minified = minify(SOURCE, remove_unused_imports=True, remove_unused_variables=True, remove_unused_definitions=True, remove_unreachable=True) + assert minified == 'import sys\ndef main():return sys.argv\ncall(main())' + + +def test_minify_with_renaming(): + minified = minify(SOURCE, remove_unused_imports=True, remove_unused_variables=True, remove_unused_definitions=True, remove_unreachable=True, rename_globals=True) + assert minified == 'import sys\ndef A():return sys.argv\ncall(A())' + + +def test_minify_after_removing_asserts(): + source = ''' +import os +def main(): + assert os + return 1 +print(main()) +''' + assert 'os' in minify(source, remove_unused_imports=True) + assert 'os' not in minify(source, remove_asserts=True, remove_unused_imports=True) + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='annotations need python 3') +def test_minify_with_annotations(): + source = ''' +import typing +def f(a: typing.List) -> None: + pass +print(f) +''' + assert 'typing' in minify(source, remove_annotations=False, remove_unused_imports=True) + assert 'typing' not in minify(source, remove_unused_imports=True) + + +def test_minify_leaves_tainted_module_alone(): + source = 'from os import *\nimport sys\nx = 1\ncall(1)' + assert minify(source, remove_unused_imports=True, remove_unused_variables=True, remove_unused_definitions=True, remove_unreachable=True) == 'from os import*\nimport sys\nx=1\ncall(1)' + + +def pyminify(*args): + result = run_subprocess([sys.executable, '-m', 'python_minifier', '-'] + list(args), timeout=30, input_data=SOURCE) + assert result.returncode == 0 + return safe_decode(result.stdout).replace('\r\n', '\n') + + +def test_cli_options_are_disabled_by_default(): + assert pyminify() == minify(SOURCE) + + +def test_cli_options(): + assert pyminify('--remove-unused-imports') == minify(SOURCE, remove_unused_imports=True) + assert pyminify('--remove-unused-variables') == minify(SOURCE, remove_unused_variables=True) + assert pyminify('--remove-unused-definitions') == minify(SOURCE, remove_unused_definitions=True) + assert pyminify('--remove-unreachable') == minify(SOURCE, remove_unreachable=True) diff --git a/test/test_remove_unreachable.py b/test/test_remove_unreachable.py new file mode 100644 index 00000000..6a580760 --- /dev/null +++ b/test/test_remove_unreachable.py @@ -0,0 +1,522 @@ +import ast +import sys + +import pytest + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unreachable(source): + module = ast.parse(source, 'remove_unreachable') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=False, remove_variables=False, remove_definitions=False, remove_unreachable=True) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unreachable(source) + compare_ast(expected_ast, actual_ast) + + +def test_after_return(): + run_test(''' +def f(): + return 1 + print('unreachable') +''', ''' +def f(): + return 1 +''') + + +def test_after_raise(): + run_test(''' +def f(): + raise ValueError() + print('unreachable') +''', ''' +def f(): + raise ValueError() +''') + + +def test_after_break_and_continue(): + run_test(''' +for i in range(10): + if i: + break + print('unreachable') + continue + print('unreachable') +else: + print('reachable') +print('reachable') +''', ''' +for i in range(10): + if i: + break + continue +else: + print('reachable') +print('reachable') +''') + + +def test_removes_all_following_statements(): + run_test(''' +def f(): + return 1 + x = 2 + def g(): + pass + class C: + pass + print(x) +''', ''' +def f(): + return 1 +''') + + +def test_module_level(): + run_test(''' +raise SystemExit(0) +print('unreachable') +''', 'raise SystemExit(0)') + + +def test_nested_suites(): + run_test(''' +def f(a): + while a: + if a: + return 1 + print('unreachable') + for i in a: + try: + raise ValueError + print('unreachable') + except ValueError: + continue + print('unreachable') + finally: + pass +''', ''' +def f(a): + while a: + if a: + return 1 + for i in a: + try: + raise ValueError + except ValueError: + continue + finally: + pass +''') + + +def test_in_class_body(): + run_test(''' +class C: + raise ValueError + x = 1 +''', ''' +class C: + raise ValueError +''') + + +def test_if_else_that_both_terminate(): + run_test(''' +def f(a): + if a: + return 1 + else: + return 2 + print('unreachable') +''', ''' +def f(a): + if a: + return 1 + else: + return 2 +''') + + run_test(''' +def f(a, b): + if a: + return 1 + elif b: + raise ValueError + else: + return 2 + print('unreachable') +''', ''' +def f(a, b): + if a: + return 1 + elif b: + raise ValueError + else: + return 2 +''') + + +def test_if_that_may_not_terminate(): + source = ''' +def f(a, b): + if a: + return 1 + print('reachable') + if a: + return 1 + else: + pass + print('reachable') + if a: + return 1 + elif b: + return 2 + print('reachable') +''' + run_test(source, source) + + +def test_try_with_handlers_that_all_terminate(): + run_test(''' +def f(): + try: + return 1 + except ValueError: + return 2 + except KeyError: + raise + print('unreachable') +''', ''' +def f(): + try: + return 1 + except ValueError: + return 2 + except KeyError: + raise +''') + + +def test_try_that_may_not_terminate(): + source = ''' +def f(): + try: + return 1 + except ValueError: + pass + print('reachable') + try: + pass + except ValueError: + return 2 + print('reachable') +''' + run_test(source, source) + + +def test_try_with_finally_that_terminates(): + run_test(''' +def f(): + try: + pass + finally: + return 1 + print('unreachable') +''', ''' +def f(): + try: + pass + finally: + return 1 +''') + + +def test_try_with_finally_that_does_not_terminate(): + source = ''' +def f(): + try: + pass + finally: + cleanup() + print('reachable') +''' + run_test(source, source) + + +def test_try_body_that_terminates_with_finally(): + run_test(''' +def f(): + try: + return 1 + finally: + cleanup() + print('unreachable') +''', ''' +def f(): + try: + return 1 + finally: + cleanup() +''') + + +def test_try_else_that_terminates(): + run_test(''' +def f(): + try: + pass + except ValueError: + return 1 + else: + return 2 + print('unreachable') +''', ''' +def f(): + try: + pass + except ValueError: + return 1 + else: + return 2 +''') + + +def test_with_that_returns(): + run_test(''' +def f(): + with open('f') as fp: + return fp.read() + print('unreachable') +''', ''' +def f(): + with open('f') as fp: + return fp.read() +''') + + +def test_with_that_raises_may_suppress_the_exception(): + source = ''' +def f(): + with suppress(ValueError): + raise ValueError + print('reachable') + with suppress(ValueError): + if a: + return 1 + else: + raise ValueError + print('reachable') +''' + run_test(source, source) + + +def test_loops_may_not_terminate(): + source = ''' +def f(a): + for i in a: + return 1 + print('reachable') + while a: + return 1 + print('reachable') + while True: + return 1 + print('reachable') +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason='match needs python 3.10') +def test_match_may_not_terminate(): + source = ''' +def f(a): + match a: + case 1: + return 1 + case _: + return 2 + print('reachable') +''' + run_test(source, source) + + +def test_yield_is_kept(): + source = ''' +def f(): + return + yield +''' + run_test(source, source) + + source = ''' +def f(): + return + x = yield +''' + run_test(source, source) + + +def test_yield_in_nested_function_is_removed(): + run_test(''' +def f(): + return + def g(): + yield +''', ''' +def f(): + return +''') + + +@pytest.mark.skipif(sys.version_info < (3, 3), reason='yield from needs python 3.3') +def test_yield_from_is_kept(): + source = ''' +def f(): + return + yield from g() +''' + run_test(source, source) + + +def test_global_declaration_is_kept(): + source = ''' +def f(): + return + global x + x = 1 +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='nonlocal needs python 3') +def test_nonlocal_declaration_is_kept(): + source = ''' +def f(): + x = 1 + def g(): + return + nonlocal x + return g +''' + run_test(source, source) + + +def test_only_assignment_to_a_local_name_is_kept(): + source = ''' +x = 1 +def f(): + return x + x = 2 +''' + run_test(source, source) + + source = ''' +x = 1 +def f(): + return x + def x(): + pass +''' + run_test(source, source) + + source = ''' +x = 1 +def f(): + return x + import x +''' + run_test(source, source) + + source = ''' +x = 1 +def f(): + def g(): + return x + return g + x = 2 +''' + run_test(source, source) + + +def test_local_name_that_is_not_used_otherwise_is_removed(): + run_test(''' +def f(): + return 1 + x = 2 + print(x) +''', ''' +def f(): + return 1 +''') + + +def test_local_name_that_is_still_bound_is_removed(): + run_test(''' +def f(): + x = 1 + return x + x = 2 +''', ''' +def f(): + x = 1 + return x +''') + + +def test_module_names_are_dynamic(): + run_test(''' +print(x) +raise SystemExit +x = 1 +''', ''' +print(x) +raise SystemExit +''') + + +def test_class_names_are_dynamic(): + run_test(''' +class C: + print(x) + raise ValueError + x = 1 +''', ''' +class C: + print(x) + raise ValueError +''') + + +def test_class_names_in_function_are_kept(): + source = ''' +def f(): + x = 1 + class C: + print(x) + raise ValueError + x = 2 + return C +''' + run_test(source, source) + + +def test_exec_taints_module(): + source = ''' +def f(): + return 1 + print('unreachable') +exec('f()') +''' + run_test(source, source) diff --git a/test/test_remove_unused_definitions.py b/test/test_remove_unused_definitions.py new file mode 100644 index 00000000..af85a86e --- /dev/null +++ b/test/test_remove_unused_definitions.py @@ -0,0 +1,297 @@ +import ast +import sys + +import pytest + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unused_definitions(source): + module = ast.parse(source, 'remove_unused_definitions') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=False, remove_variables=False, remove_definitions=True, remove_unreachable=False) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unused_definitions(source) + compare_ast(expected_ast, actual_ast) + + +def test_unused_function(): + run_test(''' +def unused(): + pass +print(1) +''', 'print(1)') + + +def test_unused_class(): + run_test(''' +class Unused: + pass +print(1) +''', 'print(1)') + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason='async functions need python 3.5') +def test_unused_async_function(): + run_test(''' +async def unused(): + pass +print(1) +''', 'print(1)') + + +def test_used_definitions(): + source = ''' +def used(): + return 1 +class Used: + pass +print(used(), Used()) +''' + run_test(source, source) + + +def test_function_passed_as_argument_is_used(): + source = ''' +def callback(): + pass +register(callback) +''' + run_test(source, source) + + +def test_class_used_as_base_is_used(): + source = ''' +class Base: + pass +class Child(Base): + pass +print(Child()) +''' + run_test(source, source) + + +def test_class_used_in_except_clause_is_used(): + source = ''' +class MyError(Exception): + pass +try: + pass +except MyError: + pass +''' + run_test(source, source) + + +def test_decorated_definitions_are_kept(): + source = ''' +@app.route('/') +def index(): + pass +@dataclass +class Config: + pass +''' + run_test(source, source) + + +def test_decorators_are_used(): + source = ''' +def decorator(f): + return f +@decorator +def main(): + pass +''' + run_test(source, source) + + +def test_dunder_definitions_are_kept(): + source = ''' +def __getattr__(name): + pass +class __Meta__: + pass +''' + run_test(source, source) + + +def test_definitions_listed_in_all_are_kept(): + run_test(''' +__all__ = ['helper', 'Helper'] +def helper(): + pass +class Helper: + pass +def other(): + pass +''', ''' +__all__ = ['helper', 'Helper'] +def helper(): + pass +class Helper: + pass +''') + + +def test_dynamic_all_keeps_every_module_definition(): + source = ''' +__all__ = names() +def helper(): + pass +''' + run_test(source, source) + + +def test_methods_are_kept(): + source = ''' +class C: + def method(self): + pass + class Inner: + pass +print(C) +''' + run_test(source, source) + + +def test_unused_function_in_function(): + run_test(''' +def outer(): + def inner(): + pass + return 1 +print(outer()) +''', ''' +def outer(): + return 1 +print(outer()) +''') + + +def test_used_function_in_function(): + source = ''' +def outer(): + def inner(): + pass + return inner +print(outer()) +''' + run_test(source, source) + + +def test_function_in_function_is_not_used_by_other_scopes(): + run_test(''' +def helper(): + pass +def outer(): + def helper(): + pass + return 1 +print(helper(), outer()) +''', ''' +def helper(): + pass +def outer(): + return 1 +print(helper(), outer()) +''') + + +def test_empty_suite_is_replaced(): + run_test(''' +def outer(): + def inner(): + pass +print(outer()) +''', ''' +def outer(): + 0 +print(outer()) +''') + + +def test_definitions_that_evaluate_calls_are_kept(): + source = ''' +def f(a=call()): + pass +class C(base()): + pass +class D: + attribute = call() +class E: + call() +''' + run_test(source, source) + + +def test_definitions_with_harmless_defaults_are_removed(): + run_test(''' +def f(a=1, *args, **kwargs): + call() +class C(Base): + x = 1 + y = 'text' + def method(self, a=2): + call() + class Inner: + pass +print(1) +''', 'print(1)') + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='keyword only arguments and metaclass keywords need python 3') +def test_definitions_with_harmless_keywords_are_removed(): + run_test(''' +def f(a, *args, b=[], **kwargs): + call() +class C(Base, metaclass=Meta): + pass +print(1) +''', 'print(1)') + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='annotations need python 3') +def test_definitions_with_annotations_that_evaluate_calls_are_kept(): + source = ''' +def f(a: call()): + pass +def g() -> call(): + pass +''' + run_test(source, source) + + run_test(''' +def f(a: int, *args: str, b: 'text' = 1, **kwargs: dict) -> None: + pass +print(1) +''', 'print(1)') + + +def test_removing_a_definition_can_make_another_unused(): + run_test(''' +def a(): + return b() +def b(): + return c() +def c(): + return 1 +print(1) +''', 'print(1)') + + +def test_exec_taints_module(): + source = ''' +def f(): + pass +exec('f()') +''' + run_test(source, source) diff --git a/test/test_remove_unused_imports.py b/test/test_remove_unused_imports.py new file mode 100644 index 00000000..9ba6a035 --- /dev/null +++ b/test/test_remove_unused_imports.py @@ -0,0 +1,295 @@ +import ast + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unused_imports(source): + module = ast.parse(source, 'remove_unused_imports') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=True, remove_variables=False, remove_definitions=False, remove_unreachable=False) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unused_imports(source) + compare_ast(expected_ast, actual_ast) + + +def test_unused_import(): + run_test(''' +import os +print(1) +''', 'print(1)') + + +def test_used_import(): + source = ''' +import os +print(os.getcwd()) +''' + run_test(source, source) + + +def test_unused_names_in_import(): + run_test(''' +import os, sys, json +print(sys.argv) +''', ''' +import sys +print(sys.argv) +''') + + +def test_unused_names_in_from_import(): + run_test(''' +from os.path import join, exists, isdir +print(join('a', 'b'), exists('c')) +''', ''' +from os.path import join, exists +print(join('a', 'b'), exists('c')) +''') + + +def test_unused_from_import_with_relative_module(): + run_test(''' +from . import sibling +from .. import parent +print(parent) +''', ''' +from .. import parent +print(parent) +''') + + +def test_unused_alias(): + run_test(''' +import numpy as np +from os import path as p +print(1) +''', 'print(1)') + + +def test_used_alias_hides_original_name(): + source = ''' +import numpy as np +from os import path as p +print(np, p) +''' + run_test(source, source) + + +def test_original_name_of_alias_is_not_a_use(): + run_test(''' +import numpy as np +print(numpy) +''', 'print(numpy)') + + +def test_dotted_import_binds_root_package(): + run_test(''' +import os.path +print(1) +''', 'print(1)') + + source = ''' +import os.path +print(os.sep) +''' + run_test(source, source) + + +def test_dotted_imports_of_same_package_are_used_together(): + source = ''' +import xml.dom +import xml.etree +print(xml.dom) +''' + run_test(source, source) + + +def test_future_import_is_kept(): + source = ''' +from __future__ import division +print(1) +''' + run_test(source, source) + + +def test_import_used_in_function(): + source = ''' +import os +def f(): + return os.getcwd() +''' + run_test(source, source) + + +def test_import_used_in_class_body(): + source = ''' +import os +class C: + path = os.getcwd() +''' + run_test(source, source) + + +def test_import_used_in_decorator_and_default(): + source = ''' +import functools +import sys +@functools.lru_cache +def f(a=sys.maxsize): + pass +''' + run_test(source, source) + + +def test_unused_import_in_function(): + run_test(''' +def f(): + import os + return 1 +''', ''' +def f(): + return 1 +''') + + +def test_function_import_is_not_used_by_other_scopes(): + run_test(''' +import os +def f(): + import os + return os.getcwd() +''', ''' +def f(): + import os + return os.getcwd() +''') + + +def test_import_in_class_body_is_kept(): + source = ''' +class C: + import os +''' + run_test(source, source) + + +def test_empty_suite_is_replaced(): + run_test(''' +def f(): + import os +''', ''' +def f(): + 0 +''') + + run_test(''' +try: + import ujson +except ImportError: + pass +''', ''' +try: + 0 +except ImportError: + pass +''') + + +def test_module_with_only_unused_imports_is_empty(): + run_test(''' +import os +from sys import argv +''', '') + + +def test_import_names_listed_in_all_are_kept(): + source = ''' +import os +import sys +__all__ = ['os'] +''' + run_test(source, ''' +import os +__all__ = ['os'] +''') + + +def test_import_names_added_to_all_are_kept(): + run_test(''' +import os +import sys +import json +__all__ = ['os'] +__all__ += ['sys'] +''', ''' +import os +import sys +__all__ = ['os'] +__all__ += ['sys'] +''') + + +def test_dynamic_all_keeps_every_import(): + source = ''' +import os +__all__ = sorted(['os']) +''' + run_test(source, source) + + source = ''' +import os +__all__ = [] +__all__.append('os') +''' + run_test(source, source) + + +def test_star_import_taints_module(): + source = ''' +from os import * +import sys +''' + run_test(source, source) + + +def test_exec_taints_module(): + source = ''' +import sys +exec('print(sys)') +''' + run_test(source, source) + + +def test_name_assigned_after_import_is_not_a_use(): + run_test(''' +import os +os = None +''', ''' +os = None +''') + + +def test_import_used_by_augmented_assignment_is_kept(): + source = ''' +import os +os += 1 +''' + run_test(source, source) + + +def test_global_declaration_is_a_use(): + source = ''' +import os +def f(): + global os + os = None +''' + run_test(source, source) diff --git a/test/test_remove_unused_variables.py b/test/test_remove_unused_variables.py new file mode 100644 index 00000000..13d503a4 --- /dev/null +++ b/test/test_remove_unused_variables.py @@ -0,0 +1,364 @@ +import ast +import sys + +import pytest + +from python_minifier.ast_annotation import add_parent +from python_minifier.ast_compare import compare_ast +from python_minifier.rename import add_namespace +from python_minifier.transforms.dead_code import eliminate_dead_code + + +def remove_unused_variables(source): + module = ast.parse(source, 'remove_unused_variables') + + add_parent(module) + add_namespace(module) + return eliminate_dead_code(module, remove_imports=False, remove_variables=True, remove_definitions=False, remove_unreachable=False) + + +def run_test(source, expected): + expected_ast = ast.parse(expected) + actual_ast = remove_unused_variables(source) + compare_ast(expected_ast, actual_ast) + + +def test_unused_variable(): + run_test(''' +x = 42 +print(1) +''', 'print(1)') + + +def test_used_variable(): + source = ''' +x = 42 +print(x) +''' + run_test(source, source) + + +def test_side_effect_free_values(): + run_test(''' +a = 'text' +b = [1, 2, {3: (4, 5)}] +c = 1 + 2 * 3 +d = not a +e = 1 if a else 2 +f = a.attribute +g = a[1:2] +h = lambda: 1 +i = len +j = a and b or c +k = a < b +print(1) +''', 'print(1)') + + +def test_values_with_side_effects_are_kept(): + source = ''' +a = call() +b = [call()] +c = {1: call()} +d = 1 + call() +e = [i for i in range(3)] +f = a.method() +g = lambda x=call(): x +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason='unpacking in displays needs python 3.5') +def test_keeps_value_that_unpacks(): + source = ''' +a = [*b] +c = {**d} +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason='f-strings and annotated assignments need python 3.6') +def test_f_string_value(): + run_test(''' +a = f'{b}' +c = f'{call()}' +print(1) +''', ''' +c = f'{call()}' +print(1) +''') + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason='async functions need python 3.5') +def test_await_and_yield_values_are_kept(): + source = ''' +async def f(): + x = await call() +def g(): + y = yield 1 +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason='annotated assignments need python 3.6') +def test_annotated_assignment(): + run_test(''' +x: int = 42 +y: int +z: int = call() +print(1) +''', ''' +y: int +z: int = call() +print(1) +''') + + source = ''' +x: call() = 1 +''' + run_test(source, source) + + +def test_chained_and_unpacking_targets_are_kept(): + source = ''' +a = b = 1 +c, d = 1, 2 +[e, f] = [1, 2] +''' + run_test(source, source) + + +def test_attribute_and_subscript_targets_are_kept(): + source = ''' +obj.x = 1 +arr[0] = 99 +''' + run_test(source, source) + + +def test_augmented_assignment_is_a_use(): + source = ''' +x = 0 +x += 1 +''' + run_test(source, source) + + +def test_del_is_a_use(): + source = ''' +x = 0 +del x +''' + run_test(source, source) + + +def test_loop_and_with_targets_are_kept(): + source = ''' +for i in range(3): + pass +with open('f') as fp: + pass +''' + run_test(source, source) + + +def test_dunder_names_are_kept(): + source = ''' +__version__ = '1.0' +__author__ = 'me' +''' + run_test(source, source) + + +def test_names_listed_in_all_are_kept(): + run_test(''' +__all__ = ['x'] +x = 1 +y = 2 +''', ''' +__all__ = ['x'] +x = 1 +''') + + +def test_dynamic_all_keeps_every_module_variable(): + source = ''' +__all__ = list_names() +x = 1 +''' + run_test(source, source) + + run_test(''' +__all__ = list_names() +def f(): + y = 1 +''', ''' +__all__ = list_names() +def f(): + 0 +''') + + +def test_variable_used_in_nested_function(): + source = ''' +x = 1 +def f(): + return x +''' + run_test(source, source) + + +def test_unused_variable_in_function(): + run_test(''' +def f(): + x = 1 + return 2 +''', ''' +def f(): + return 2 +''') + + +def test_function_variable_is_not_used_by_other_scopes(): + run_test(''' +x = 1 +def f(): + x = 2 + return 3 +print(x) +''', ''' +x = 1 +def f(): + return 3 +print(x) +''') + + run_test(''' +x = 1 +def f(): + x = 2 + return 3 +''', ''' +def f(): + return 3 +''') + + +def test_variable_used_by_inner_function(): + source = ''' +def f(): + x = 1 + def g(): + return x + return g +''' + run_test(source, source) + + +@pytest.mark.skipif(sys.version_info < (3, 0), reason='nonlocal needs python 3') +def test_nonlocal_declaration_is_a_use(): + source = ''' +def f(): + x = 1 + def g(): + nonlocal x + x = 2 + return g +''' + run_test(source, source) + + +def test_global_declaration_is_a_use(): + source = ''' +x = 1 +def f(): + global x + x = 2 +''' + run_test(source, source) + + +def test_class_attributes_are_kept(): + source = ''' +class C: + x = 1 +''' + run_test(source, source) + + +def test_variable_in_class_method(): + run_test(''' +class C: + def method(self): + x = 1 + return 2 +''', ''' +class C: + def method(self): + return 2 +''') + + +def test_variable_in_class_in_function_is_kept(): + source = ''' +def f(): + class C: + x = 1 + return C +''' + run_test(source, source) + + +def test_empty_suite_is_replaced(): + run_test(''' +def f(): + x = 1 +''', ''' +def f(): + 0 +''') + + run_test(''' +if a: + x = 1 +else: + y = 2 +''', ''' +if a: + 0 +else: + 0 +''') + + +def test_module_with_only_unused_variables_is_empty(): + run_test(''' +a = 1 +b = 2 +''', '') + + +def test_removing_a_variable_can_make_another_unused(): + run_test(''' +a = 1 +b = a +c = b + 1 +print(1) +''', 'print(1)') + + +def test_locals_taints_module(): + source = ''' +def f(): + x = 1 + return locals() +''' + run_test(source, source) + + +def test_star_import_taints_module(): + source = ''' +from os import * +x = 1 +''' + run_test(source, source)