From aaf37f807d46f1988806adb3cab2d862fc32be0c Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 30 Aug 2026 21:03:41 +0000 Subject: [PATCH 1/8] [Python] Prototype a Cling-native Python-to-C++ transpiler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ROOT._pydeclare, a JIT-free alternative to ROOT.Numba.Declare: the decorated Python callable is translated to C++ source and given to cling, instead of being compiled by numba and reached through a raw function pointer baked into the generated wrapper. Two backends discover the operations, and share one lowering layer, so they emit identical C++ for input both accept: ast walks the Python syntax tree; translates control flow trace runs the callable once on symbolic arguments; refuses control flow rather than silently baking in one Python run Anything outside the supported subset is refused at declaration time with the offending Python source line, rather than approximated: silently wrong numbers are worse than an error. The places where Python and C++ genuinely disagree -- true division, floor division, modulo of negatives, negative indices, round-half-to-even, summing booleans -- are handled by explicit templates in the C++ support library rather than left to type inference. ROOT.Py.Declare is the new entry point, emitting into the Py namespace. ROOT.Numba.Declare is routed through a dispatcher so that $ROOT_NUMBA_DECLARE_BACKEND=ast|trace re-runs existing code, including the df.Define(name, callable, cols) pythonization, on a backend of choice with the "Numba::f(x)" call strings unchanged. test/pydeclare.py is numbadeclare.py parameterised over the three implementations, registered three times in CTest. All 46 tests pass on each; the only skip is the one assertion about numba's own refcounts. 🤖 Done with the help of AI --- .../pythonizations/python/CMakeLists.txt | 7 + .../pythonizations/python/ROOT/_facade.py | 18 +- .../python/ROOT/_pydeclare/README.md | 98 ++ .../python/ROOT/_pydeclare/__init__.py | 252 ++++ .../python/ROOT/_pydeclare/ast_backend.py | 1056 +++++++++++++++++ .../python/ROOT/_pydeclare/cpptypes.py | 456 +++++++ .../python/ROOT/_pydeclare/emit.py | 678 +++++++++++ .../python/ROOT/_pydeclare/errors.py | 65 + .../python/ROOT/_pydeclare/support.py | 692 +++++++++++ .../python/ROOT/_pydeclare/trace_backend.py | 652 ++++++++++ .../python/ROOT/_pythonization/_rdf_pyz.py | 4 +- .../pyroot/pythonizations/test/CMakeLists.txt | 8 + .../pyroot/pythonizations/test/pydeclare.py | 798 +++++++++++++ 13 files changed, 4780 insertions(+), 4 deletions(-) create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py create mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py create mode 100644 bindings/pyroot/pythonizations/test/pydeclare.py diff --git a/bindings/pyroot/pythonizations/python/CMakeLists.txt b/bindings/pyroot/pythonizations/python/CMakeLists.txt index 18ffa3f1330fb..33496b9644516 100644 --- a/bindings/pyroot/pythonizations/python/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/python/CMakeLists.txt @@ -83,6 +83,13 @@ set(py_sources ROOT/_jupyroot/magics/jsrootmagic.py ROOT/_jupyroot/magics/rootbrowsemagic.py ROOT/_numbadeclare.py + ROOT/_pydeclare/__init__.py + ROOT/_pydeclare/ast_backend.py + ROOT/_pydeclare/cpptypes.py + ROOT/_pydeclare/emit.py + ROOT/_pydeclare/errors.py + ROOT/_pydeclare/support.py + ROOT/_pydeclare/trace_backend.py ROOT/_pythonization/__init__.py ROOT/_pythonization/_cppinstance.py ROOT/_pythonization/_drawables.py diff --git a/bindings/pyroot/pythonizations/python/ROOT/_facade.py b/bindings/pyroot/pythonizations/python/ROOT/_facade.py index e8467adcc1074..a8205b18c9e92 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_facade.py @@ -534,14 +534,28 @@ def TMVA(self): # Create and overload Numba namespace @property def Numba(self): - from ._numbadeclare import _NumbaDeclareDecorator + from ._pydeclare import _numba_declare_dispatch self._cppyy.cppdef("namespace Numba {}") ns = self._fallback_getattr("Numba") - ns.Declare = staticmethod(_NumbaDeclareDecorator) + # The implementation behind the decorator is selected by + # $ROOT_NUMBA_DECLARE_BACKEND: 'numba' (default), 'ast' or 'trace'. + ns.Declare = staticmethod(_numba_declare_dispatch) del type(self).Numba return ns + # Create and overload the Py namespace, holding the transpiler based + # counterpart of Numba.Declare + @property + def Py(self): + from ._pydeclare import Declare + + self._cppyy.cppdef("namespace Py {}") + ns = self._fallback_getattr("Py") + ns.Declare = staticmethod(Declare) + del type(self).Py + return ns + @property def NumbaExt(self): import numba diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md new file mode 100644 index 0000000000000..75cb0573f5618 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md @@ -0,0 +1,98 @@ +# Translating Python callables to C++ + +A JIT-free alternative to `ROOT.Numba.Declare`. The decorated callable is +translated to C++ *source* and handed to cling, rather than compiled by numba +and reached through a raw function pointer. + +```python +@ROOT.Py.Declare(["float", "int"], "float") +def pypow(x, y): + return x**y + +ROOT.RDataFrame(4).Define("x", "(float)rdfentry_").Define("y", "Py::pypow(x, 3)") +``` + +The generated code is on the callable as `__cpp_wrapper__`. + +## The two backends + +Both lower through the same module (`emit.py`), so for input they both accept +they emit identical C++. + +| | `ast` (default) | `trace` | +|---|---|---| +| How | walks the Python syntax tree | runs the callable once on symbolic arguments | +| Needs the source | yes | no | +| `if` / `for` / `while` / early return | translated | refused | +| `not x`, `x and y`, `x or y` | translated | refused (resolved via `__bool__`) | +| Operations it has never heard of | refused by name | often works, C++ decides | + +Select with `backend=`, or `$ROOT_PYDECLARE_BACKEND`. The tracer refuses +control flow because it cannot see it: a branch is either invisible (it +depended on an argument) or silently baked in (it did not). Pass `unroll=True` +to accept the second case deliberately. + +`ROOT.Numba.Declare` can be routed to either backend with +`$ROOT_NUMBA_DECLARE_BACKEND=ast|trace`, which keeps existing `"Numba::f(x)"` +call strings working. + +## The supported subset + +**Types.** `bool`, the integer types, `float`, `double`; `RVec`, `std::vector` +and `std::array` of them (`std::vector` and `std::array` are viewed as an +`RVec` inside the body, without copying); and any other C++ class, which is +opaque -- you can call its methods and read its members, and the C++ compiler +resolves them. + +**Expressions.** Arithmetic, comparisons (including chained ones on scalars), +`and`/`or`/`not`, the bitwise operators, indexing with wrap-around for negative +indices, slicing with full Python semantics, boolean-mask indexing (`v[v > 0]`), +conditional expressions, calls to other declared callables and to cppyy +entities such as `ROOT.TMath.Abs`. + +**Statements** (`ast` backend only): assignment, augmented and annotated +assignment, `if`/`elif`/`else`, `for` over a range or an array, `while`, +`break`, `continue`, `pass`, `assert`, early `return`. + +**Builtins.** `abs`, `bool`, `float`, `int`, `len`, `max`, `min`, `pow`, +`round`, `sum`. + +**numpy and math.** The elementwise maths functions, the reductions (`.sum()`, +`.mean()`, `.std()`, `.min()`, `.max()`, `.argmin()`, `.argmax()`, `.any()`, +`.all()`, `.prod()`, and their free-function spellings), `np.where`, +`np.array([...])`, `.astype(...)`, `np.pi` and friends. + +**Constants.** Any expression built only from module-level or closure variables +is evaluated once at declaration time and inlined, as numba freezes globals. +Only numpy, math and the pure builtins are allowed to run during that +evaluation, so a call like `os.getpid()` is refused rather than frozen. + +## Where Python and C++ disagree + +The generated code is explicit about every place the two languages differ, via +templates in `support.py`: + +| Python | C++ | What is emitted | +|---|---|---| +| `7 / 2` is `3.5` | integer division | `PyD::Div` | +| `-7 // 2` is `-4` | truncates to `-3` | `PyD::FloorDiv` | +| `-7 % 2` is `1` | `-1` | `PyD::Mod` (skipped for unsigned, where they agree) | +| `2 ** 10` is an `int` | `std::pow` returns a double | `PyD::Pow` | +| `v[-1]` is the last element | out of bounds | `PyD::Index` | +| `round(0.5)` is `0` | `std::round` gives `1` | `PyD::Round` (half to even) | +| `sum` of a bool array is an `int` | `Sum` of `RVec` is a `bool` | `PyD::Sum` | + +## Deliberate restrictions + +Local variables are declared `auto`, so the type is always the one the C++ +compiler deduces, never one this package guessed. A variable may not change +type between assignments, and one assigned inside a nested block is hoisted to +function scope -- which is Python's own scoping -- and therefore needs a type +that can be determined up front; annotate it (`total: 'double' = 0.0`) if it +cannot be. + +Not supported, and refused with a message pointing at the line: comprehensions, +lambdas inside the body, dicts, sets, tuples, strings, `try`, `with`, `global`, +generators, f-strings, `*args`, multi-dimensional indexing, and `np.random` +(a C++ generator would not reproduce numpy's stream, and silently different +random numbers are worse than an error). diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py new file mode 100644 index 0000000000000..d0285611720d4 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py @@ -0,0 +1,252 @@ +# Author: Jonas Rembser CERN 08/2026 + +################################################################################ +# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +"""Declare C++ functions from Python callables by translating them to C++. + +This is a JIT-free alternative to ``ROOT.Numba.Declare``: instead of compiling +the Python callable with numba and calling into it through a raw function +pointer, the callable is translated to C++ source and handed to cling. The +generated code is therefore inlinable into the RDataFrame event loop, can call +anything cling knows about (including user classes), and needs no third-party +dependency. + +Two backends are available and produce the same C++ for the same input: + +``ast`` + Walks the Python syntax tree. Handles control flow. + +``trace`` + Runs the callable once with symbolic arguments and records the operations. + Cannot see control flow, and says so rather than guessing. + +Usage:: + + @ROOT.Py.Declare(["float", "int"], "float") + def pypow(x, y): + return x**y + + ROOT.RDataFrame(4).Define("x", "(float)rdfentry_").Define("y", "Py::pypow(x, 3)") + +The generated C++ is available on the callable as ``__cpp_wrapper__``. +""" + +import inspect +import os + +from . import cpptypes as ct +from . import emit, support +from .ast_backend import CPP_KEYWORDS, AstTranspiler +from .errors import PyDeclareError +from .trace_backend import trace as _trace + +__all__ = ["Declare", "PyDeclareError", "declare"] + +BACKENDS = ("ast", "trace") +DEFAULT_NAMESPACE = "Py" + +#: (namespace, name) -> generated code, so that re-running a script in the same +#: session does not hit a cling redefinition error. +_DECLARED = {} + + +def _default_backend(): + backend = os.environ.get("ROOT_PYDECLARE_BACKEND", "ast").strip().lower() + if backend not in BACKENDS: + raise PyDeclareError("ROOT_PYDECLARE_BACKEND is '{}'; expected one of {}".format(backend, ", ".join(BACKENDS))) + return backend + + +def _sanitize(name): + if name in CPP_KEYWORDS or name.startswith("PyD"): + return name + "_" + return name + + +def _param_declaration(raw, type_, cpp_name): + """The C++ parameter declaration for one argument. + + Fundamental types are taken by value. Everything else is taken by const + reference unless the user explicitly wrote a reference in the signature, in + which case that spelling is honoured. + """ + if type_.kind == ct.FUND: + return "{} {}".format(type_.cpp(), cpp_name) + raw_s = str(raw).strip() + if raw_s.endswith("&"): + qualifier = "const " if raw_s.startswith("const") else "" + return "{}{} &{}".format(qualifier, type_.cpp(), cpp_name) + return "const {} &{}".format(type_.cpp(), cpp_name) + + +def _signature_names(func, n_expected): + try: + signature = inspect.signature(func) + except (TypeError, ValueError): + raise PyDeclareError("Cannot inspect the signature of {!r}".format(func)) + names = [] + for param in signature.parameters.values(): + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD): + raise PyDeclareError("*args and **kwargs are not supported") + if param.kind == param.KEYWORD_ONLY: + raise PyDeclareError("Keyword-only arguments are not supported") + names.append(param.name) + if len(names) != n_expected: + raise PyDeclareError( + "The callable takes {} argument(s) but {} input type(s) were declared".format(len(names), n_expected) + ) + return names + + +def declare(func, input_types, return_type=None, name=None, namespace=DEFAULT_NAMESPACE, backend=None, unroll=False): + """Translate a Python callable to C++ and declare it to cling. + + Returns the original callable, with the generated code and some metadata + attached as attributes. + """ + backend = (backend or _default_backend()).strip().lower() + if backend not in BACKENDS: + raise PyDeclareError("Unknown backend '{}'; expected one of {}".format(backend, ", ".join(BACKENDS))) + + raw_types = list(input_types or []) + param_names = _signature_names(func, len(raw_types)) + parsed = [ct.parse_type(t) for t in raw_types] + declared_return = ct.parse_type(return_type) if return_type is not None else None + if declared_return is not None and declared_return.is_unknown: + declared_return = None + + cpp_params = [_sanitize(n) for n in param_names] + declarations = [_param_declaration(raw_types[i], parsed[i], cpp_params[i]) for i in range(len(parsed))] + + # std::vector and std::array are viewed as RVec inside the body, so that the + # element-wise operators are available for all supported containers. + adapters = [] + body_types = [] + body_names = [] + for i, t in enumerate(parsed): + if t.is_container and t.container != "rvec": + view = cpp_params[i] + "_v" + adapters.append(" const auto {} = {}::AsRVec({});".format(view, emit.PYD, cpp_params[i])) + body_names.append(view) + body_types.append(ct.rvec_of(t.scalar())) + else: + body_names.append(cpp_params[i]) + body_types.append(t) + + if backend == "ast": + transpiler = AstTranspiler(func, param_names, body_types, declared_return, body_names) + body, deduced = transpiler.translate() + else: + body, deduced = _trace(func, param_names, body_types, declared_return, body_names, unroll=unroll) + + cpp_return = (declared_return or deduced).cpp() if (declared_return or deduced) is not None else "auto" + func_name = name or getattr(func, "__name__", None) + if not func_name or func_name == "": + raise PyDeclareError("Anonymous callables need an explicit name=... argument") + + code = _render(namespace, func_name, cpp_return, declarations, adapters, body, func, backend) + + key = (namespace, func_name) + previous = _DECLARED.get(key) + if previous is None: + support.ensure_declared() + import ROOT + + if not ROOT.gInterpreter.Declare(code): + raise PyDeclareError("cling rejected the generated code:\n\n{}".format(code)) + _DECLARED[key] = code + elif previous != code: + raise PyDeclareError( + "'{}::{}' was already declared from a different Python callable in this session.\n" + " Pass name=... to give this one a different C++ name.".format(namespace, func_name) + ) + + func.__cpp_wrapper__ = code + func.__pydeclare_cpp_name__ = "{}::{}".format(namespace, func_name) + func.__pydeclare_return_type__ = declared_return or deduced + func.__pydeclare_backend__ = backend + return func + + +def _render(namespace, func_name, cpp_return, declarations, adapters, body, func, backend): + origin = "" + try: + origin = " ({}:{})".format(os.path.basename(inspect.getsourcefile(func) or "?"), func.__code__.co_firstlineno) + except (OSError, TypeError, AttributeError): + pass + header = [ + "namespace {} {{".format(namespace), + "", + "namespace PyD = ROOT::Internal::PyDeclare;", + "", + "/// Generated from the Python callable '{}'{} by the ROOT".format( + getattr(func, "__name__", func_name), origin + ), + "/// Python-to-C++ transpiler, {} backend.".format(backend), + "{} {}({})".format(cpp_return, func_name, ", ".join(declarations)), + "{", + ] + return "\n".join(header + adapters + [body, "}", "", "}} // namespace {}".format(namespace)]) + "\n" + + +def Declare(input_types=None, return_type=None, name=None, namespace=DEFAULT_NAMESPACE, backend=None, unroll=False): + """Decorator making a Python callable available in C++, by translation. + + Arguments mirror ``ROOT.Numba.Declare``: + + input_types + List of C++ type names, one per argument of the callable. + return_type + The C++ return type. If omitted, it is deduced -- either by the + transpiler, or by the C++ compiler through an ``auto`` return type. + name + The name of the generated C++ function; defaults to the Python name. + namespace + The C++ namespace to declare it in; defaults to ``Py``. + backend + ``"ast"`` or ``"trace"``; defaults to $ROOT_PYDECLARE_BACKEND or ast. + unroll + Tracing backend only: accept Python-level control flow, evaluating it + once at declaration time. + """ + + def inner(func): + return declare( + func, + input_types, + return_type=return_type, + name=name, + namespace=namespace, + backend=backend, + unroll=unroll, + ) + + return inner + + +def _numba_declare_dispatch(input_types, return_type=None, name=None, **kwargs): + """``ROOT.Numba.Declare``, routed to whichever backend is selected. + + ``ROOT_NUMBA_DECLARE_BACKEND`` picks the implementation: ``numba`` (the + default, the original implementation), ``ast`` or ``trace``. The generated + function always lands in the ``Numba`` C++ namespace, so existing code that + says ``"Numba::myfunc(x)"`` keeps working whichever backend is in use. + """ + backend = os.environ.get("ROOT_NUMBA_DECLARE_BACKEND", "numba").strip().lower() + if backend in ("", "numba"): + from .._numbadeclare import _NumbaDeclareDecorator + + return _NumbaDeclareDecorator(input_types, return_type, name) + if backend not in BACKENDS: + raise PyDeclareError( + "ROOT_NUMBA_DECLARE_BACKEND is '{}'; expected 'numba', {}".format( + backend, " or ".join(repr(b) for b in BACKENDS) + ) + ) + return Declare(input_types, return_type, name, namespace="Numba", backend=backend, **kwargs) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py new file mode 100644 index 0000000000000..414323779dbfe --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py @@ -0,0 +1,1056 @@ +# Author: Jonas Rembser CERN 08/2026 + +################################################################################ +# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +"""The AST-traversal backend. + +Walks the Python syntax tree of the decorated callable and emits C++. Because +the argument types are known -- RDataFrame knows its column types, and the +decorator takes them explicitly -- forward type inference through the tree is +enough to choose the right lowering for every operation. + +Compared to the tracing backend this one sees the *source*, so it can translate +control flow: if/elif/else, for, while, break/continue and early returns. +""" + +import ast +import inspect +import textwrap + +from . import cpptypes as ct +from . import emit +from .cpptypes import UNKNOWN_T +from .errors import PyDeclareError, raise_unsupported + +CPP_KEYWORDS = frozenset( + """ + alignas alignof and and_eq asm auto bitand bitor bool break case catch char char8_t char16_t + char32_t class compl concept const consteval constexpr constinit const_cast continue co_await + co_return co_yield decltype default delete do double dynamic_cast else enum explicit export + extern false float for friend goto if inline int long mutable namespace new noexcept not not_eq + nullptr operator or or_eq private protected public register reinterpret_cast requires return + short signed sizeof static static_assert static_cast struct switch template this thread_local + throw true try typedef typeid typename union unsigned using virtual void volatile wchar_t while + xor xor_eq NULL + """.split() +) + +BUILTIN_CASTS = {"float", "int", "bool"} + +#: Builtins that are safe to run at declaration time when folding constants. +PURE_BUILTINS = frozenset( + ["abs", "bool", "float", "int", "len", "max", "min", "pow", "round", "sum", "range", "list", "tuple"] +) + + +class _ModuleRef: + """A reference to numpy or math in the callable's enclosing scope.""" + + def __init__(self, kind, path): + self.kind = kind # "numpy" or "math" + self.path = path + + +class _FuncRef: + """A numpy/math function that has a C++ counterpart.""" + + def __init__(self, name, path): + self.name = name + self.path = path + + +class _CppRef: + """A cppyy entity (class, namespace or free function) usable from C++.""" + + def __init__(self, cpp_name): + self.cpp_name = cpp_name + + +class _DeclaredRef: + """Another callable that was already translated by this package.""" + + def __init__(self, cpp_name, return_type): + self.cpp_name = cpp_name + self.return_type = return_type + + +class _Scope: + """One lexical block; variables declared here die at the closing brace.""" + + def __init__(self, parent=None): + self.parent = parent + self.names = {} + + def lookup(self, name): + scope = self + while scope is not None: + if name in scope.names: + return scope.names[name] + scope = scope.parent + return None + + +class AstTranspiler: + """Translate one Python callable into the body of a C++ function.""" + + def __init__(self, func, param_names, param_types, declared_return, cpp_param_names): + self.func = func + self.param_names = param_names + self.param_types = param_types + self.declared_return = declared_return + self.cpp_param_names = cpp_param_names + + self.lines = [] + self.indent = 1 + self.depth = 0 + self.tmp_counter = 0 + self.flat_scope = False + self.hoisted = {} # python name -> (cpp name, CppType) + self.collected = {} # python name -> CppType, filled by the first pass + self.collected_depth = {} # python name -> set of block depths it is assigned at + self.return_types = [] + self.loop_depth = 0 + self.globals_cache = None + + # -- error helpers ------------------------------------------------------ + + def fail_at(self, node): + def _fail(msg, hint=None): + raise_unsupported(self.func, node, msg, hint) + + return _fail + + # -- name helpers ------------------------------------------------------- + + def cpp_name(self, name): + if name in CPP_KEYWORDS or name.startswith("PyD"): + return name + "_" + return name + + def tmp(self, stem="t"): + self.tmp_counter += 1 + return "_pyd_{}{}".format(stem, self.tmp_counter) + + def write(self, text): + self.lines.append(" " * self.indent + text) + + # ----------------------------------------------------------------------- + # Entry point + # ----------------------------------------------------------------------- + + def translate(self): + tree = self._parse() + body = tree.body + + # Pass 1: a flat scope, so that every read resolves; this only collects + # the types of variables that will have to be hoisted to function scope. + self.flat_scope = True + saved = (self.lines, self.indent) + self.lines = [] + self._run(body) + self.lines, self.indent = saved + + hoist_names = {n for n, depths in self.collected_depth.items() if any(d > 0 for d in depths)} + + # Pass 2: the real one. + self.flat_scope = False + self.return_types = [] + self.tmp_counter = 0 + self.hoisted = {} + for name in sorted(hoist_names): + t = self.collected.get(name, UNKNOWN_T) + if t is None or t.is_unknown: + raise PyDeclareError( + "Cannot determine the C++ type of the variable '{}', which is assigned inside a " + "nested block and therefore has to be declared up front.\n" + " Give it a type by annotating the assignment, for example " + "\"{}: 'double' = 0.0\", or assign it once before the block.".format(name, name) + ) + self.hoisted[name] = (self.cpp_name(name), t) + self.write("{} {}{{}};".format(t.cpp(), self.cpp_name(name))) + + self._run(body) + + if not self.return_types: + raise PyDeclareError( + "The callable '{}' has no reachable 'return' statement; " + "a translated function must return a value.".format(getattr(self.func, "__name__", "")) + ) + + return "\n".join(self.lines), self._deduce_return_type() + + def _deduce_return_type(self): + if self.declared_return is not None: + return self.declared_return + types = [t for t in self.return_types if t is not None and not t.is_unknown] + if len(types) == len(self.return_types) and types and all(t == types[0] for t in types): + return types[0] + return None # emit 'auto' and let the compiler deduce it + + def _run(self, body): + self.scope = _Scope() + for i, pname in enumerate(self.param_names): + self.scope.names[pname] = emit.Value(self.cpp_param_names[i], self.param_types[i]) + # Variables hoisted to function scope are visible from the first line on, + # which is what makes Python's function-wide variable scope work here. + for pname, (cpp, type_) in self.hoisted.items(): + self.scope.names[pname] = emit.Value(cpp, type_) + self.exec_block(body, new_scope=False) + + def _parse(self): + try: + src = inspect.getsource(self.func) + except (OSError, TypeError): + raise PyDeclareError( + "Cannot read the source of the callable. The AST backend needs the Python source; " + "callables defined in the interpreter prompt or in an exec() string cannot be " + "translated. Use the tracing backend, or move the function into a file." + ) + src = textwrap.dedent(src) + try: + mod = ast.parse(src) + except SyntaxError as exc: + raise PyDeclareError("Cannot parse the source of the callable: {}".format(exc)) + + node = mod.body[0] + if isinstance(node, ast.FunctionDef): + self._check_args(node.args) + return node + # A decorated lambda arrives as an assignment or a bare expression. + for sub in ast.walk(mod): + if isinstance(sub, ast.Lambda): + self._check_args(sub.args) + return ast.Module(body=[ast.Return(value=sub.body)], type_ignores=[]) + raise PyDeclareError("Expected a function definition or a lambda, got {}".format(type(node).__name__)) + + def _check_args(self, args): + if args.vararg is not None or args.kwarg is not None: + raise PyDeclareError("*args and **kwargs are not supported") + if args.kwonlyargs: + raise PyDeclareError("Keyword-only arguments are not supported") + names = [a.arg for a in args.posonlyargs] + [a.arg for a in args.args] + if names != list(self.param_names): + raise PyDeclareError( + "The declared signature does not match the callable's parameters: expected {}, got {}".format( + list(self.param_names), names + ) + ) + + # ----------------------------------------------------------------------- + # Statements + # ----------------------------------------------------------------------- + + def exec_block(self, body, new_scope=True): + if new_scope and not self.flat_scope: + self.scope = _Scope(self.scope) + for stmt in body: + self.exec_stmt(stmt) + if new_scope and not self.flat_scope: + self.scope = self.scope.parent + + def exec_stmt(self, node): + handler = getattr(self, "st_" + type(node).__name__, None) + if handler is None: + raise_unsupported( + self.func, + node, + "The statement '{}' is not supported".format(type(node).__name__), + hint="Supported statements are: return, assignment (including augmented and " + "annotated), if/elif/else, for, while, break, continue, pass, assert and " + "expression statements.", + ) + handler(node) + + def st_Pass(self, node): + pass + + def st_Expr(self, node): + # Docstrings and other bare constants are dropped. + if isinstance(node.value, ast.Constant): + return + value = self.expr(node.value) + self.write("{};".format(value.code)) + + def st_Return(self, node): + if node.value is None: + raise_unsupported(self.func, node, "'return' without a value is not supported") + value = self.expr(node.value) + self.return_types.append(value.type) + if self.declared_return is not None: + self.write("return {};".format(self._converted(value, self.declared_return))) + else: + self.write("return {};".format(value.code)) + + def st_Assign(self, node): + if len(node.targets) != 1: + raise_unsupported(self.func, node, "Chained assignment (a = b = c) is not supported") + self._assign_to(node.targets[0], self.expr(node.value), node) + + def st_AnnAssign(self, node): + if node.value is None: + raise_unsupported(self.func, node, "A type annotation without a value is not supported") + if not isinstance(node.target, ast.Name): + raise_unsupported(self.func, node, "Annotated assignment is only supported for plain variables") + spelling = self._annotation_text(node.annotation) + declared = ct.parse_type(spelling) + value = self.expr(node.value) + name = node.target.id + cpp = self.cpp_name(name) + self._record(name, declared) + if name in self.hoisted: + self.write("{} = {};".format(cpp, self._converted(value, declared))) + else: + existing = self.scope.lookup(name) + if existing is not None: + raise_unsupported(self.func, node, "Variable '{}' is already declared".format(name)) + self.write("{} {} = {};".format(declared.cpp(), cpp, self._converted(value, declared))) + self.scope.names[name] = emit.Value(cpp, declared) + + def _annotation_text(self, node): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.Name): + return node.id + raise_unsupported( + self.func, + node, + "Type annotations must be C++ type names given as strings", + hint="For example: x: 'double' = 0.0", + ) + + def st_AugAssign(self, node): + op = _BINOP_NAMES.get(type(node.op).__name__) + if op is None: + raise_unsupported(self.func, node, "Augmented assignment with this operator is not supported") + current = self.expr(_as_load(node.target)) + result = emit.binop(op, current, self.expr(node.value), self.fail_at(node)) + self._assign_to(node.target, result, node) + + def _record(self, name, type_): + """Remember an assignment for the hoisting pass.""" + self.collected_depth.setdefault(name, set()).add(self.depth) + prev = self.collected.get(name) + if prev is None: + self.collected[name] = type_ + elif prev != type_: + # Two different types for one name: keep it unknown so that the + # hoisting pass reports it rather than picking one at random. + self.collected[name] = UNKNOWN_T if (prev.is_unknown or type_.is_unknown) else _unify(prev, type_) + + def _converted(self, value, target_type): + """The expression, converted to target_type unless it already has it.""" + if value.type == target_type: + return value.code + return "{}::Return<{}>({})".format(emit.PYD, target_type.cpp(), value.code) + + def _assign_to(self, target, value, node): + if isinstance(target, ast.Subscript): + obj = self.expr(target.value) + index = self.expr(target.slice) + slot = emit.subscript(obj, index, self.fail_at(node)) + self.write("{} = {};".format(slot.code, value.code)) + return + if isinstance(target, (ast.Tuple, ast.List)): + raise_unsupported(self.func, node, "Tuple unpacking is not supported") + if not isinstance(target, ast.Name): + raise_unsupported(self.func, node, "Cannot assign to {}".format(type(target).__name__)) + + name = target.id + cpp = self.cpp_name(name) + self._record(name, value.type) + + if name in self.hoisted: + declared = self.hoisted[name][1] + self.write("{} = {};".format(cpp, self._converted(value, declared))) + return + + existing = self.scope.lookup(name) + if existing is None: + self.write("auto {} = {};".format(cpp, value.code)) + self.scope.names[name] = emit.Value(cpp, value.type) + else: + if not (existing.type.is_unknown or value.type.is_unknown) and existing.type != value.type: + raise_unsupported( + self.func, + node, + "Variable '{}' changes type from '{}' to '{}'".format(name, existing.type, value.type), + hint="A translated variable keeps the type it was first assigned; C++ has no " + "dynamically typed variables. Use a second variable instead.", + ) + self.write("{} = {};".format(cpp, value.code)) + + def st_If(self, node): + cond = emit.truth(self.expr(node.test), self.fail_at(node)) + self.write("if ({}) {{".format(cond.code)) + self.indent += 1 + self.depth += 1 + self.exec_block(node.body) + self.depth -= 1 + self.indent -= 1 + if node.orelse: + self.write("} else {") + self.indent += 1 + self.depth += 1 + self.exec_block(node.orelse) + self.depth -= 1 + self.indent -= 1 + self.write("}") + + def st_While(self, node): + if node.orelse: + raise_unsupported(self.func, node, "'while ... else' is not supported") + cond = emit.truth(self.expr(node.test), self.fail_at(node)) + self.write("while ({}) {{".format(cond.code)) + self.indent += 1 + self.depth += 1 + self.loop_depth += 1 + self.exec_block(node.body) + self.loop_depth -= 1 + self.depth -= 1 + self.indent -= 1 + self.write("}") + + def st_For(self, node): + if node.orelse: + raise_unsupported(self.func, node, "'for ... else' is not supported") + if not isinstance(node.target, ast.Name): + raise_unsupported( + self.func, + node, + "Only a single loop variable is supported", + hint="enumerate() and zip() are not translated; loop over an index range instead.", + ) + name = node.target.id + cpp = self.cpp_name(name) + + if isinstance(node.iter, ast.Call) and self._is_range(node.iter.func): + self._for_range(node, name, cpp) + return + + iterable = self.expr(node.iter) + if not iterable.type.is_container and not iterable.type.is_unknown: + raise_unsupported( + self.func, + node, + "Cannot iterate over a value of type '{}'".format(iterable.type), + hint="Only arrays and range() can be iterated.", + ) + elem = iterable.type.scalar() + self._record(name, elem) + self.write("for (auto {} : {}) {{".format(cpp, iterable.code)) + self._loop_body(node, name, cpp, elem) + + def _is_range(self, func): + return isinstance(func, ast.Name) and func.id == "range" and self.scope.lookup("range") is None + + def _for_range(self, node, name, cpp): + args = [self.expr(a) for a in node.iter.args] + if node.iter.keywords or not 1 <= len(args) <= 3: + raise_unsupported(self.func, node, "range() takes one to three positional arguments") + start = emit.Value("0", ct.LONG_T) if len(args) == 1 else args[0] + stop = args[0] if len(args) == 1 else args[1] + step = args[2] if len(args) == 3 else emit.Value("1", ct.LONG_T) + + if step.code.lstrip("-").isdigit(): + step_value = int(step.code) + else: + raise_unsupported( + self.func, + node, + "The step of range() must be an integer literal", + hint="A runtime step would need a direction check on every iteration; write two " + "loops, or use a while loop.", + ) + if step_value == 0: + raise_unsupported(self.func, node, "range() step must not be zero") + + bound = self.tmp("stop") + self.write("{{ const long {} = static_cast({});".format(bound, stop.code)) + self.indent += 1 + cmp_op = "<" if step_value > 0 else ">" + incr = ( + "{} += {}".format(cpp, step_value) + if abs(step_value) != 1 + else ("++" + cpp if step_value > 0 else "--" + cpp) + ) + self.write( + "for (long {} = static_cast({}); {} {} {}; {}) {{".format(cpp, start.code, cpp, cmp_op, bound, incr) + ) + self._record(name, ct.LONG_T) + self._loop_body(node, name, cpp, ct.LONG_T) + self.indent -= 1 + self.write("}") + + def _loop_body(self, node, name, cpp, elem_type): + self.indent += 1 + self.depth += 1 + self.loop_depth += 1 + if not self.flat_scope: + self.scope = _Scope(self.scope) + self.scope.names[name] = emit.Value(cpp, elem_type) + for stmt in node.body: + self.exec_stmt(stmt) + if not self.flat_scope: + self.scope = self.scope.parent + self.loop_depth -= 1 + self.depth -= 1 + self.indent -= 1 + self.write("}") + + def st_Break(self, node): + if not self.loop_depth: + raise_unsupported(self.func, node, "'break' outside a loop") + self.write("break;") + + def st_Continue(self, node): + if not self.loop_depth: + raise_unsupported(self.func, node, "'continue' outside a loop") + self.write("continue;") + + def st_Assert(self, node): + cond = emit.truth(self.expr(node.test), self.fail_at(node)) + message = "assertion failed in {}".format(getattr(self.func, "__name__", "")) + if node.msg is not None and isinstance(node.msg, ast.Constant) and isinstance(node.msg.value, str): + message = node.msg.value + self.write('if (!({})) {{ throw std::runtime_error("{}"); }}'.format(cond.code, message.replace('"', '\\"'))) + + # ----------------------------------------------------------------------- + # Expressions + # ----------------------------------------------------------------------- + + def expr(self, node): + """Translate an expression node, or fail loudly.""" + # An expression that does not depend on any argument is a compile-time + # constant: evaluate it in Python and inline the result, exactly like + # numba freezes globals. + folded = self._try_fold(node) + if folded is not None: + return folded + + handler = getattr(self, "ex_" + type(node).__name__, None) + if handler is None: + raise_unsupported(self.func, node, "The expression '{}' is not supported".format(type(node).__name__)) + result = handler(node) + if not isinstance(result, emit.Value): + raise_unsupported( + self.func, + node, + "'{}' cannot be used as a value here".format(ast.dump(node)[:60]), + ) + return result + + def ex_Constant(self, node): + return emit.literal(node.value, self.fail_at(node)) + + def ex_Name(self, node): + local = self.scope.lookup(node.id) + if local is not None: + return local + resolved = self._resolve_global(node.id, node) + if isinstance(resolved, emit.Value): + return resolved + return resolved # a reference object; the caller decides what to do + + def ex_BinOp(self, node): + op = _BINOP_NAMES.get(type(node.op).__name__) + if op is None: + raise_unsupported(self.func, node, "Binary operator not supported") + return emit.binop(op, self.expr(node.left), self.expr(node.right), self.fail_at(node)) + + def ex_UnaryOp(self, node): + op = {"UAdd": "+", "USub": "-", "Not": "not", "Invert": "~"}.get(type(node.op).__name__) + if op is None: + raise_unsupported(self.func, node, "Unary operator not supported") + return emit.unaryop(op, self.expr(node.operand), self.fail_at(node)) + + def ex_BoolOp(self, node): + op = "and" if isinstance(node.op, ast.And) else "or" + return emit.boolop(op, [self.expr(v) for v in node.values], self.fail_at(node)) + + def ex_Compare(self, node): + fail = self.fail_at(node) + ops = [_CMP_NAMES.get(type(o).__name__) for o in node.ops] + if any(o is None for o in ops): + raise_unsupported(self.func, node, "Comparison operator not supported") + operands = [self.expr(node.left)] + [self.expr(c) for c in node.comparators] + if len(ops) == 1: + return emit.compare(ops[0], operands[0], operands[1], fail) + if any(v.type.is_container for v in operands): + raise_unsupported( + self.func, + node, + "Chained comparisons are not supported for arrays", + hint="numpy raises here too; combine two comparisons with '&'.", + ) + parts = [emit.compare(ops[i], operands[i], operands[i + 1], fail) for i in range(len(ops))] + return emit.boolop("and", parts, fail) + + def ex_IfExp(self, node): + return emit.ternary(self.expr(node.test), self.expr(node.body), self.expr(node.orelse), self.fail_at(node)) + + def ex_Subscript(self, node): + obj = self.expr(node.value) + if isinstance(node.slice, ast.Slice): + sl = node.slice + start = self.expr(sl.lower) if sl.lower is not None else None + stop = self.expr(sl.upper) if sl.upper is not None else None + step = self.expr(sl.step) if sl.step is not None else None + return emit.slice_(obj, start, stop, step, self.fail_at(node)) + if isinstance(node.slice, ast.Tuple): + raise_unsupported(self.func, node, "Multi-dimensional indexing is not supported") + return emit.subscript(obj, self.expr(node.slice), self.fail_at(node)) + + def ex_Attribute(self, node): + base = self._maybe_ref(node.value) + name = node.attr + + if isinstance(base, _ModuleRef): + return self._module_attribute(base, name, node) + if isinstance(base, _CppRef): + return _CppRef("{}::{}".format(base.cpp_name, name)) + if isinstance(base, _DeclaredRef): + raise_unsupported(self.func, node, "Cannot take an attribute of a declared function") + + value = base + if value.type.is_container: + if name in emit.ARRAY_METHODS: + return _BoundMethod(value, name) + if name == "size": + return emit.length(value, self.fail_at(node)) + if name == "astype": + return _BoundMethod(value, "astype") + raise_unsupported( + self.func, + node, + "Arrays have no attribute '{}'".format(name), + hint="Supported: {}.".format(", ".join(sorted(emit.ARRAY_METHODS)) + ", size"), + ) + if value.type.is_fund: + raise_unsupported(self.func, node, "'{}' has no attribute '{}'".format(value.type, name)) + # Opaque C++ object: pass the access straight through. + return _CppAttr(value, name) + + def ex_Call(self, node): + fail = self.fail_at(node) + emit.check_no_kwargs(node.keywords, "a translated function", fail) + # An unshadowed builtin name is dispatched by name; its arguments are + # translated first, so that an unsupported argument reports itself. + if ( + isinstance(node.func, ast.Name) + and self.scope.lookup(node.func.id) is None + and node.func.id not in self._get_globals() + ): + return self._builtin_call(node.func.id, [self.expr(a) for a in node.args], node) + + callee = self._maybe_ref(node.func) + + # np.array([...]) builds an array out of element expressions, and + # astype() takes a dtype rather than a value; both need their arguments + # before the ordinary translation. + if isinstance(callee, _FuncRef) and callee.name == "array": + return self._array_call(node, fail) + if isinstance(callee, _BoundMethod) and callee.name == "astype": + if len(node.args) != 1: + raise_unsupported(self.func, node, "astype() takes exactly one argument") + return emit.astype(callee.obj, self._static_value(node.args[0], "astype()"), fail) + + args = [self.expr(a) for a in node.args] + + if isinstance(callee, _BoundMethod): + return self._array_method_call(callee, args, node) + if isinstance(callee, _CppAttr): + return emit.cpp_method(callee.obj, callee.name, args) + if isinstance(callee, _CppRef): + return emit.call(callee.cpp_name, args, UNKNOWN_T) + if isinstance(callee, _DeclaredRef): + return emit.call(callee.cpp_name, args, callee.return_type or UNKNOWN_T) + if isinstance(callee, _FuncRef): + return self._math_call(callee, args, node) + if isinstance(callee, _ModuleRef): + raise_unsupported(self.func, node, "'{}' is a module and cannot be called".format(callee.path)) + + # A plain name: builtins. + if isinstance(node.func, ast.Name): + return self._builtin_call(node.func.id, args, node) + raise_unsupported(self.func, node, "This call cannot be translated") + + def _static_value(self, node, what): + """Evaluate an argument that must be known at declaration time.""" + if not self._is_static(node) or not self._fold_safe(node): + raise_unsupported( + self.func, + node, + "The argument of {} has to be known when the function is declared".format(what), + ) + try: + return eval(compile(ast.Expression(body=node), "", "eval"), dict(self._get_globals())) + except Exception as exc: + raise_unsupported(self.func, node, "Cannot evaluate the argument of {}: {}".format(what, exc)) + + def _array_call(self, node, fail): + """Translate np.array([a, b, ...]).""" + if len(node.args) != 1: + raise_unsupported(self.func, node, "np.array() takes exactly one argument here") + arg = node.args[0] + if not isinstance(arg, (ast.List, ast.Tuple)): + raise_unsupported( + self.func, + node, + "np.array() needs a list or tuple of elements here", + hint="A constant array is folded automatically; a dynamic one is built from its " + "elements, as in np.array([x, y]).", + ) + return emit.array_from_elements([self.expr(e) for e in arg.elts], fail) + + def _array_method_call(self, bound, args, node): + return emit.array_method(bound.obj, bound.name, args, self.fail_at(node)) + + def _math_call(self, ref, args, node): + fail = self.fail_at(node) + name = ref.name + if name in emit.UNARY_FUNCTIONS and len(args) == 1: + return emit.unary_function(emit.UNARY_FUNCTIONS[name], args[0], fail) + if name in emit.BINARY_FUNCTIONS and len(args) == 2: + return emit.binary_function(emit.BINARY_FUNCTIONS[name], args[0], args[1], fail) + if name in emit.FREE_REDUCTIONS and len(args) == 1: + if not args[0].type.is_container: + raise_unsupported(self.func, node, "'{}' expects an array".format(ref.path)) + return emit.array_method(args[0], _REDUCTION_METHOD[emit.FREE_REDUCTIONS[name]], [], fail) + if name == "where" and len(args) == 3: + return emit.ternary(args[0], args[1], args[2], fail) + if name in ("sqrt", "fabs") and len(args) == 1: + return emit.unary_function(emit.UNARY_FUNCTIONS[name], args[0], fail) + raise_unsupported( + self.func, + node, + "'{}' is not available in the supported subset (or was called with {} arguments)".format( + ref.path, len(args) + ), + ) + + def _builtin_call(self, name, args, node): + fail = self.fail_at(node) + if name in BUILTIN_CASTS and len(args) == 1: + return emit.cast(name, args[0], fail) + if name == "len" and len(args) == 1: + return emit.length(args[0], fail) + if name == "abs" and len(args) == 1: + return emit.unary_function("Abs", args[0], fail) + if name == "round" and len(args) == 1: + return emit.unary_function("Round", args[0], fail) + if name == "pow" and len(args) == 2: + return emit.binop("**", args[0], args[1], fail) + if name in ("min", "max"): + if len(args) == 1: + if not args[0].type.is_container: + raise_unsupported(self.func, node, "{}() of a single value needs an array".format(name)) + return emit.array_method(args[0], name, [], fail) + if len(args) == 2: + if args[0].type.is_container or args[1].type.is_container: + raise_unsupported( + self.func, + node, + "{}() of two arrays is ambiguous".format(name), + hint="Use np.{}imum(a, b) for the element-wise version.".format(name), + ) + return emit.binary_function("Maximum" if name == "max" else "Minimum", args[0], args[1], fail) + if name == "sum" and len(args) == 1: + if not args[0].type.is_container: + raise_unsupported(self.func, node, "sum() expects an array") + return emit.array_method(args[0], "sum", [], fail) + if name == "range": + raise_unsupported(self.func, node, "range() can only be used in a for loop") + raise_unsupported( + self.func, + node, + "The function '{}' is not available in the supported subset".format(name), + hint="Supported builtins: abs, bool, float, int, len, max, min, pow, round, sum.", + ) + + def ex_List(self, node): + raise_unsupported( + self.func, + node, + "List literals are only supported when they do not depend on the arguments", + hint="A constant list is folded into an RVec literal; a list built from arguments " + "would need a dynamic container.", + ) + + ex_Tuple = ex_List + ex_Dict = ex_List + ex_Set = ex_List + + def ex_ListComp(self, node): + raise_unsupported( + self.func, + node, + "List comprehensions are not supported", + hint="Use element-wise array expressions, or an explicit for loop.", + ) + + ex_SetComp = ex_ListComp + ex_DictComp = ex_ListComp + ex_GeneratorExp = ex_ListComp + + def ex_Lambda(self, node): + raise_unsupported(self.func, node, "Nested lambdas are not supported") + + def ex_JoinedStr(self, node): + raise_unsupported(self.func, node, "f-strings are not supported") + + def ex_Starred(self, node): + raise_unsupported(self.func, node, "Argument unpacking is not supported") + + # ----------------------------------------------------------------------- + # Names from the enclosing scope + # ----------------------------------------------------------------------- + + def _maybe_ref(self, node): + """Like expr(), but may also return a reference object.""" + if isinstance(node, ast.Name): + local = self.scope.lookup(node.id) + if local is not None: + return local + folded = self._try_fold(node) + if folded is not None: + return folded + return self._resolve_global(node.id, node) + if isinstance(node, ast.Attribute): + return self.ex_Attribute(node) + return self.expr(node) + + def _get_globals(self): + if self.globals_cache is None: + env = {} + env.update(getattr(self.func, "__globals__", {}) or {}) + closure = getattr(self.func, "__closure__", None) + code = getattr(self.func, "__code__", None) + if closure and code: + for name, cell in zip(code.co_freevars, closure): + try: + env[name] = cell.cell_contents + except ValueError: + pass + self.globals_cache = env + return self.globals_cache + + def _resolve_global(self, name, node): + env = self._get_globals() + if name not in env: + import builtins + + if hasattr(builtins, name): + raise_unsupported( + self.func, + node, + "The builtin '{}' is not available in the supported subset".format(name), + ) + raise_unsupported(self.func, node, "Undefined name '{}'".format(name)) + return self._classify(env[name], name, node) + + def _classify(self, obj, path, node): + value = emit.constant_value(obj, self.fail_at(node)) + if value is not None: + return value + + module_name = getattr(obj, "__name__", None) + if inspect.ismodule(obj): + if module_name in ("numpy", "math"): + return _ModuleRef("numpy" if module_name == "numpy" else "math", path) + raise_unsupported( + self.func, + node, + "Module '{}' cannot be used in translated code".format(module_name), + hint="Only numpy and math are understood.", + ) + + declared = getattr(obj, "__pydeclare_cpp_name__", None) + if declared is not None: + return _DeclaredRef(declared, getattr(obj, "__pydeclare_return_type__", None)) + + cpp_name = _cppyy_name(obj) + if cpp_name is not None: + return _CppRef(cpp_name) + + if callable(obj) and module_name in emit.UNARY_FUNCTIONS: + return _FuncRef(module_name, path) + + raise_unsupported( + self.func, + node, + "Cannot use '{}' (a {}) in translated code".format(path, type(obj).__name__), + hint="Values captured from the enclosing scope must be numbers, arrays of numbers, " + "C++ entities known to cling, or other declared callables.", + ) + + def _module_attribute(self, ref, name, node): + if name in emit.CONSTANTS: + code, type_ = emit.CONSTANTS[name] + return emit.Value(code, type_) + known = ( + name in emit.UNARY_FUNCTIONS + or name in emit.BINARY_FUNCTIONS + or name in emit.FREE_REDUCTIONS + or name in ("where", "array") + ) + if known: + return _FuncRef(name, "{}.{}".format(ref.path, name)) + raise_unsupported( + self.func, + node, + "'{}.{}' is not available in the supported subset".format(ref.path, name), + ) + + # ----------------------------------------------------------------------- + # Constant folding + # ----------------------------------------------------------------------- + + def _is_static(self, node): + """True if the expression only involves names from the enclosing scope.""" + for sub in ast.walk(node): + if isinstance(sub, ast.Name): + if self.scope.lookup(sub.id) is not None: + return False + if sub.id not in self._get_globals(): + return False + if isinstance(sub, (ast.Lambda, ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)): + return False + return True + + def _callee_is_safe(self, func_node): + """True if this callee is a numpy/math function or a pure builtin.""" + if isinstance(func_node, ast.Name) and func_node.id not in self._get_globals(): + return func_node.id in PURE_BUILTINS + try: + code = compile(ast.Expression(body=func_node), "", "eval") + obj = eval(code, dict(self._get_globals())) # noqa: S307 + except Exception: + return False + module = (getattr(obj, "__module__", "") or "").split(".")[0] + if module in ("numpy", "math"): + return True + # A bound method of a numpy object, as in np.array([...]).astype(...) + receiver = getattr(obj, "__self__", None) + if receiver is not None: + return (type(receiver).__module__ or "").split(".")[0] == "numpy" + return False + + def _fold_safe(self, node): + """True if evaluating this expression at declaration time is harmless. + + Freezing a constant is only sound for pure numerical code. Evaluating + an arbitrary call would silently bake in whatever it returned at import + time, which for something like os.getpid() is plainly wrong, so only + numpy, math and the pure builtins are allowed to run. + """ + for child in ast.iter_child_nodes(node): + if not self._fold_safe(child): + return False + if isinstance(node, ast.Call): + return self._callee_is_safe(node.func) + return True + + def _try_fold(self, node): + if isinstance(node, ast.Constant): + return None # handled directly, no need to compile anything + if not self._is_static(node): + return None + if not self._fold_safe(node): + return None + try: + code = compile(ast.Expression(body=node), "", "eval") + obj = eval(code, dict(self._get_globals())) # noqa: S307 + except Exception: + return None + return emit.constant_value(obj, self.fail_at(node)) + + +class _BoundMethod: + def __init__(self, obj, name): + self.obj = obj + self.name = name + + +class _CppAttr(emit.Value): + """Attribute access on an opaque C++ object. + + It is a Value in its own right, so that reading a data member works, and it + also carries the receiver so that a following call becomes a method call. + """ + + __slots__ = ("obj", "name") + + def __init__(self, obj, name): + super().__init__("{}.{}".format(obj.paren(emit.PREC_PRIMARY), name), UNKNOWN_T, emit.PREC_PRIMARY) + self.obj = obj + self.name = name + + +_BINOP_NAMES = { + "Add": "+", + "Sub": "-", + "Mult": "*", + "Div": "/", + "FloorDiv": "//", + "Mod": "%", + "Pow": "**", + "LShift": "<<", + "RShift": ">>", + "BitOr": "|", + "BitAnd": "&", + "BitXor": "^", + "MatMult": "@", +} + +_CMP_NAMES = { + "Eq": "==", + "NotEq": "!=", + "Lt": "<", + "LtE": "<=", + "Gt": ">", + "GtE": ">=", + "Is": "is", + "IsNot": "is not", + "In": "in", + "NotIn": "not in", +} + +_REDUCTION_METHOD = {fn: name for name, (fn, _) in emit.ARRAY_METHODS.items()} + + +def _as_load(node): + """A copy of an assignment target usable as an expression.""" + clone = ast.parse(ast.unparse(node), mode="eval").body if hasattr(ast, "unparse") else node + ast.copy_location(clone, node) + for sub in ast.walk(clone): + ast.copy_location(sub, node) + return clone + + +def _unify(a, b): + """The type both a and b can be stored in, or unknown.""" + if a == b: + return a + if a.is_container != b.is_container: + return UNKNOWN_T + merged = ct.promote(a, b) + return merged + + +def _cppyy_name(obj): + """The C++ name of a cppyy class, namespace or function, if it is one.""" + name = getattr(obj, "__cpp_name__", None) + if isinstance(name, str) and name: + return name + module = getattr(obj, "__module__", "") or "" + qual = getattr(obj, "__name__", None) + if isinstance(qual, str) and qual and (module.startswith("cppyy") or module == "ROOT"): + prefix = module.split(".", 1)[1] if module.startswith("cppyy.gbl.") else "" + return "{}::{}".format(prefix, qual) if prefix else qual + return None diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py new file mode 100644 index 0000000000000..eeb603eda06c4 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py @@ -0,0 +1,456 @@ +# Author: Jonas Rembser CERN 08/2026 + +################################################################################ +# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +"""A small semantic type model shared by both transpiler backends. + +The model deliberately does *not* try to reproduce the C++ type system. The +generated code declares local variables as ``auto``, so the real types are +always the ones the C++ compiler deduces. This model exists only to decide +which *rewrite* to apply -- for example whether ``x.sum()`` must become +``Sum(x)``, whether ``a | b`` is a logical or a bitwise or, or whether a value +needs Python's floor-division semantics. Being approximate here can produce a +compile error, never a silently wrong number. +""" + +import re + +from .errors import PyDeclareError + +# --------------------------------------------------------------------------- +# The type model +# --------------------------------------------------------------------------- + +FUND = "fund" # bool, int, double, ... +CONTAINER = "container" # RVec, std::vector, std::array +OPAQUE = "opaque" # any other C++ class known to cling, e.g. PtEtaPhiMVector +UNKNOWN = "unknown" # we could not infer it; emit auto and let C++ decide +VOID = "void" + + +class CppType: + """A semantic type: a kind, plus a C++ spelling for the leaf types.""" + + __slots__ = ("kind", "name", "elem", "container", "size") + + def __init__(self, kind, name=None, elem=None, container=None, size=None): + self.kind = kind + self.name = name + self.elem = elem + self.container = container + self.size = size + + # -- predicates --------------------------------------------------------- + + @property + def is_container(self): + return self.kind == CONTAINER + + @property + def is_fund(self): + return self.kind == FUND + + @property + def is_unknown(self): + return self.kind == UNKNOWN + + @property + def is_bool(self): + """True for bool scalars and for containers of bool.""" + if self.kind == FUND: + return self.name == "bool" + if self.kind == CONTAINER: + return self.elem is not None and self.elem.is_bool + return False + + @property + def is_integral(self): + leaf = self.elem if self.kind == CONTAINER else self + return leaf is not None and leaf.kind == FUND and leaf.name in INTEGRAL_TYPES + + @property + def is_floating(self): + leaf = self.elem if self.kind == CONTAINER else self + return leaf is not None and leaf.kind == FUND and leaf.name in FLOATING_TYPES + + @property + def is_signed(self): + leaf = self.elem if self.kind == CONTAINER else self + return leaf is not None and leaf.kind == FUND and leaf.name not in UNSIGNED_TYPES + + @property + def is_numeric(self): + return self.is_integral or self.is_floating + + def scalar(self): + """The element type for containers, self otherwise.""" + if self.kind == CONTAINER: + return self.elem if self.elem is not None else UNKNOWN_T + return self + + # -- spelling ----------------------------------------------------------- + + def cpp(self): + """The canonical C++ spelling of this type.""" + if self.kind in (FUND, OPAQUE): + return self.name + if self.kind == VOID: + return "void" + if self.kind == CONTAINER: + inner = self.elem.cpp() if self.elem is not None else "double" + if self.container == "array": + return "std::array<{}, {}>".format(inner, self.size) + if self.container == "vector": + return "std::vector<{}>".format(inner) + return "ROOT::VecOps::RVec<{}>".format(inner) + raise PyDeclareError("Cannot spell an unknown type in C++") + + def __repr__(self): + if self.kind == UNKNOWN: + return "" + try: + return self.cpp() + except PyDeclareError: + return "<{}>".format(self.kind) + + def __eq__(self, other): + if not isinstance(other, CppType): + return NotImplemented + return ( + self.kind == other.kind + and self.name == other.name + and self.container == other.container + and self.size == other.size + and self.elem == other.elem + ) + + def __hash__(self): + return hash((self.kind, self.name, self.container, self.size)) + + +# --------------------------------------------------------------------------- +# Fundamental types +# --------------------------------------------------------------------------- + +# Canonical spelling -> conversion rank. Used only to pick a result type for +# mixed-type arithmetic; the C++ compiler has the final word. +INTEGRAL_RANKS = { + "bool": 0, + "char": 1, + "signed char": 1, + "unsigned char": 1, + "short": 2, + "unsigned short": 2, + "int": 3, + "unsigned int": 4, + "long": 5, + "unsigned long": 6, + "long long": 7, + "unsigned long long": 8, +} +FLOATING_RANKS = {"float": 20, "double": 21, "long double": 22} + +INTEGRAL_TYPES = frozenset(INTEGRAL_RANKS) +FLOATING_TYPES = frozenset(FLOATING_RANKS) +UNSIGNED_TYPES = frozenset( + ["bool", "unsigned char", "unsigned short", "unsigned int", "unsigned long", "unsigned long long"] +) + +# Spellings the user may write, normalised to the canonical form above. +_FUND_ALIASES = { + "bool": "bool", + "_Bool": "bool", + "char": "char", + "signed char": "signed char", + "unsigned char": "unsigned char", + "short": "short", + "short int": "short", + "signed short": "short", + "unsigned short": "unsigned short", + "unsigned short int": "unsigned short", + "int": "int", + "signed": "int", + "signed int": "int", + "unsigned": "unsigned int", + "unsigned int": "unsigned int", + "long": "long", + "long int": "long", + "signed long": "long", + "unsigned long": "unsigned long", + "unsigned long int": "unsigned long", + "long long": "long long", + "long long int": "long long", + "unsigned long long": "unsigned long long", + "unsigned long long int": "unsigned long long", + "float": "float", + "double": "double", + "long double": "long double", + # ROOT typedefs that show up as RDataFrame column types + "Bool_t": "bool", + "Char_t": "char", + "UChar_t": "unsigned char", + "Short_t": "short", + "UShort_t": "unsigned short", + "Int_t": "int", + "UInt_t": "unsigned int", + "Long_t": "long", + "ULong_t": "unsigned long", + "Long64_t": "long long", + "ULong64_t": "unsigned long long", + "Float_t": "float", + "Double_t": "double", + "size_t": "unsigned long", + "std::size_t": "unsigned long", + "ROOT::RDF::RSampleInfo": None, # not fundamental, falls through to opaque +} + +BOOL_T = CppType(FUND, "bool") +INT_T = CppType(FUND, "int") +LONG_T = CppType(FUND, "long") +ULONG_T = CppType(FUND, "unsigned long") +LLONG_T = CppType(FUND, "long long") +ULLONG_T = CppType(FUND, "unsigned long long") +FLOAT_T = CppType(FUND, "float") +DOUBLE_T = CppType(FUND, "double") +UNKNOWN_T = CppType(UNKNOWN) +VOID_T = CppType(VOID) + + +def fund(name): + return CppType(FUND, name) + + +def rvec_of(elem): + """An RVec with the given element type.""" + return CppType(CONTAINER, elem=elem, container="rvec") + + +# Backwards-compatible short name used in a few places. +rvec = rvec_of + + +# --------------------------------------------------------------------------- +# Parsing C++ type spellings +# --------------------------------------------------------------------------- + +_RVEC_ALIASES = { + "RVecB": "bool", + "RVecC": "char", + "RVecD": "double", + "RVecF": "float", + "RVecI": "int", + "RVecL": "long", + "RVecLL": "long long", + "RVecU": "unsigned int", + "RVecUL": "unsigned long", + "RVecULL": "unsigned long long", +} + +_CONTAINER_HEADS = { + "RVec": "rvec", + "ROOT::RVec": "rvec", + "VecOps::RVec": "rvec", + "ROOT::VecOps::RVec": "rvec", + "vector": "vector", + "std::vector": "vector", + "array": "array", + "std::array": "array", +} + + +def _strip_cv_ref(s): + s = s.strip() + while s.endswith("&") or s.endswith("*"): + if s.endswith("*"): + raise PyDeclareError("Pointer types are not supported as arguments: '{}'".format(s)) + s = s[:-1].strip() + # Remove leading/trailing const and volatile + changed = True + while changed: + changed = False + for kw in ("const", "volatile"): + if s.startswith(kw + " "): + s = s[len(kw) + 1 :].strip() + changed = True + if s.endswith(" " + kw): + s = s[: -len(kw) - 1].strip() + changed = True + return s + + +def _split_template_args(s): + """Split 'int, 3' or 'RVec, 2' on top-level commas.""" + out, depth, cur = [], 0, "" + for ch in s: + if ch in "<([": + depth += 1 + elif ch in ">)]": + depth -= 1 + if ch == "," and depth == 0: + out.append(cur.strip()) + cur = "" + else: + cur += ch + if cur.strip(): + out.append(cur.strip()) + return out + + +def parse_type(spelling): + """Parse a C++ type spelling into a CppType. + + Understands fundamental types (and the usual ROOT typedefs), RVec, its + short aliases (RVecF, ...), std::vector and std::array, and treats anything + else as an opaque C++ class. + """ + if spelling is None: + return UNKNOWN_T + if isinstance(spelling, CppType): + return spelling + + s = _strip_cv_ref(str(spelling)) + if not s: + return UNKNOWN_T + if s == "void": + return VOID_T + + # Whitespace normalisation: collapse runs of spaces + s = re.sub(r"\s+", " ", s) + + if s in _FUND_ALIASES and _FUND_ALIASES[s] is not None: + return CppType(FUND, _FUND_ALIASES[s]) + + # The short aliases are spelled both bare and namespace-qualified. + alias = s.split("::")[-1] if s.startswith(("ROOT::", "ROOT::VecOps::")) else s + if alias in _RVEC_ALIASES: + return CppType(CONTAINER, elem=CppType(FUND, _RVEC_ALIASES[alias]), container="rvec") + + m = re.match(r"^([\w:]+)\s*<(.*)>$", s) + if m: + head, args = m.group(1), m.group(2) + kind = _CONTAINER_HEADS.get(head) + if kind is not None: + parts = _split_template_args(args) + if not parts: + raise PyDeclareError("Cannot parse container type '{}'".format(spelling)) + elem = parse_type(parts[0]) + size = None + if kind == "array": + if len(parts) < 2: + raise PyDeclareError("std::array needs a size: '{}'".format(spelling)) + size = parts[1] + return CppType(CONTAINER, elem=elem, container=kind, size=size) + # Some other template: opaque + return CppType(OPAQUE, s) + + return CppType(OPAQUE, s) + + +# --------------------------------------------------------------------------- +# Promotion +# --------------------------------------------------------------------------- + + +def _rank(t): + if t.kind != FUND: + return None + if t.name in INTEGRAL_RANKS: + return INTEGRAL_RANKS[t.name] + return FLOATING_RANKS.get(t.name) + + +def int_op_type(a, b): + """The scalar type an integer operation on *a* and *b* is carried out in. + + Mirrors ROOT::Internal::PyDeclare::IntOpType: Python has no unsigned + integers, so a mix of a signed and an unsigned operand is worked out in a + signed 64-bit type rather than converting the signed side. + """ + sa, sb = a.scalar(), b.scalar() + if not (sa.is_integral and sb.is_integral): + return promote_scalar(sa, sb) + if sa.is_signed or sb.is_signed: + return LLONG_T + return promote_scalar(sa, sb) + + +def acc_type(elem): + """The scalar type numpy accumulates a sum or product of *elem* into.""" + if not elem.is_integral: + return elem + return LLONG_T if (elem.is_signed or elem.is_bool) else ULLONG_T + + +def promote_scalar(a, b): + """Usual-arithmetic-conversions-ish result of combining two scalar types.""" + if a.kind == UNKNOWN or b.kind == UNKNOWN: + return UNKNOWN_T + if a.kind != FUND or b.kind != FUND: + return UNKNOWN_T + ra, rb = _rank(a), _rank(b) + if ra is None or rb is None: + return UNKNOWN_T + # bool and char never survive arithmetic: Python/C++ both promote to int + winner = a if ra >= rb else b + if _rank(winner) < INTEGRAL_RANKS["int"]: + return INT_T + return winner + + +def promote(a, b): + """Result type of an elementwise binary operation between a and b.""" + if a.kind == UNKNOWN or b.kind == UNKNOWN: + return UNKNOWN_T + a_cont, b_cont = a.is_container, b.is_container + if not a_cont and not b_cont: + return promote_scalar(a, b) + elem = promote_scalar(a.scalar(), b.scalar()) + # The result of mixing containers is always an RVec: that is what all the + # ROOT::VecOps operators return. + return CppType(CONTAINER, elem=elem, container="rvec") + + +def with_scalar(t, scalar_type): + """Same shape as t, but with the given scalar/element type.""" + if t.is_container: + return CppType(CONTAINER, elem=scalar_type, container="rvec") + return scalar_type + + +def bool_like(t): + """bool for scalars, RVec for containers -- the type of a comparison.""" + return with_scalar(t, BOOL_T) + + +# --------------------------------------------------------------------------- +# numpy dtype <-> C++ +# --------------------------------------------------------------------------- + +NUMPY_TO_CPP = { + "bool": "bool", + "bool_": "bool", + "int8": "char", + "uint8": "unsigned char", + "int16": "short", + "uint16": "unsigned short", + "int32": "int", + "uint32": "unsigned int", + "int64": "long", + "uint64": "unsigned long", + "intp": "long", + "float32": "float", + "float64": "double", + "float_": "double", + "double": "double", + "single": "float", + "int_": "long", + "intc": "int", + "uintc": "unsigned int", + "longlong": "long long", +} diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py new file mode 100644 index 0000000000000..aaafbd6655cd0 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py @@ -0,0 +1,678 @@ +# Author: Jonas Rembser CERN 08/2026 + +################################################################################ +# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +"""Lowering of Python operations to C++ expressions. + +This module is shared by both backends: the AST walker and the tracer both +reduce to calls into the functions here, so the two produce identical C++ for +identical Python. A backend contributes only the *discovery* of the operations +(walking a syntax tree versus recording overloaded operators); the semantics of +each individual operation live here, exactly once. +""" + +from . import cpptypes as ct +from .cpptypes import BOOL_T, DOUBLE_T, LONG_T, UNKNOWN_T, CppType +from .errors import PyDeclareError + +#: Namespace alias emitted into every generated snippet. +PYD = "PyD" +PYD_FULL = "ROOT::Internal::PyDeclare" + +# C++ operator precedence; larger binds tighter. +PREC_PRIMARY = 17 +PREC_UNARY = 16 +PREC_MUL = 14 +PREC_ADD = 13 +PREC_SHIFT = 11 +PREC_REL = 10 +PREC_EQ = 9 +PREC_BITAND = 8 +PREC_BITXOR = 7 +PREC_BITOR = 6 +PREC_AND = 5 +PREC_OR = 4 +PREC_COND = 3 + +_BINOP_PREC = { + "*": PREC_MUL, + "/": PREC_MUL, + "%": PREC_MUL, + "+": PREC_ADD, + "-": PREC_ADD, + "<<": PREC_SHIFT, + ">>": PREC_SHIFT, + "<": PREC_REL, + ">": PREC_REL, + "<=": PREC_REL, + ">=": PREC_REL, + "==": PREC_EQ, + "!=": PREC_EQ, + "&": PREC_BITAND, + "^": PREC_BITXOR, + "|": PREC_BITOR, + "&&": PREC_AND, + "||": PREC_OR, +} + + +class Value: + """A translated C++ expression together with its inferred semantic type.""" + + __slots__ = ("code", "type", "prec") + + def __init__(self, code, type_=UNKNOWN_T, prec=PREC_PRIMARY): + self.code = code + self.type = type_ if type_ is not None else UNKNOWN_T + self.prec = prec + + def paren(self, min_prec): + """This expression's code, parenthesised if it binds looser than min_prec.""" + return "({})".format(self.code) if self.prec < min_prec else self.code + + def __repr__(self): + return "Value({!r}, {})".format(self.code, self.type) + + +def call(name, args, type_=UNKNOWN_T): + """A function call expression; always a primary, never needs parentheses.""" + return Value("{}({})".format(name, ", ".join(a.code for a in args)), type_, PREC_PRIMARY) + + +def _pyd(fn, args, type_=UNKNOWN_T): + return call("{}::{}".format(PYD, fn), args, type_) + + +def _binary(op, a, b, type_): + prec = _BINOP_PREC[op] + # All the operators handled here are left-associative. + return Value("{} {} {}".format(a.paren(prec), op, b.paren(prec + 1)), type_, prec) + + +# --------------------------------------------------------------------------- +# Literals +# --------------------------------------------------------------------------- + + +def literal(value, fail): + """Translate a Python constant to a C++ literal.""" + if isinstance(value, bool): + return Value("true" if value else "false", BOOL_T) + if isinstance(value, int): + if -(2**31) <= value < 2**31: + return Value(repr(value), ct.INT_T) + if -(2**63) <= value < 2**63: + return Value("{}LL".format(value), ct.fund("long long")) + fail("Integer literal {} does not fit in a 64 bit C++ integer".format(value)) + if isinstance(value, float): + if value != value: + return Value("std::numeric_limits::quiet_NaN()", DOUBLE_T) + if value in (float("inf"), float("-inf")): + sign = "-" if value < 0 else "" + return Value("{}std::numeric_limits::infinity()".format(sign), DOUBLE_T) + text = repr(value) + if "e" not in text and "E" not in text and "." not in text and "inf" not in text: + text += ".0" + return Value(text, DOUBLE_T) + if isinstance(value, complex): + fail("Complex numbers are not supported") + if value is None: + fail("None is not supported; every branch must return a value") + if isinstance(value, str): + fail("String values are not supported") + fail("Cannot translate a constant of type {}".format(type(value).__name__)) + + +def sequence_literal(values, elem_type, fail): + """Translate a Python/numpy sequence of numbers to an RVec literal.""" + parts = [] + for v in values: + parts.append(literal(v, fail).code) + ctype = CppType(ct.CONTAINER, elem=elem_type, container="rvec") + return Value("{}{{{}}}".format(ctype.cpp(), ", ".join(parts)), ctype, PREC_PRIMARY) + + +# --------------------------------------------------------------------------- +# Binary operators +# --------------------------------------------------------------------------- + + +def _both_boolish(a, b): + return a.type.is_bool and b.type.is_bool + + +def _check_numeric(op, a, b, fail): + for v in (a, b): + if v.type.kind == ct.OPAQUE: + fail( + "Operator '{}' is not supported for values of C++ type '{}'".format(op, v.type.name), + hint="Only fundamental types and RVec/std::vector/std::array of them take part " + "in arithmetic. Call a method on the object instead.", + ) + + +def binop(op, a, b, fail): + """Lower a Python binary operator (ast.BinOp) to C++.""" + if op in ("+", "-", "*"): + _check_numeric(op, a, b, fail) + return _binary(op, a, b, ct.promote(a.type, b.type)) + + if op == "/": + _check_numeric(op, a, b, fail) + res = ct.with_scalar(ct.promote(a.type, b.type), DOUBLE_T) + if a.type.is_floating or b.type.is_floating: + # C++ already promotes to floating point, so '/' is true division. + return _binary("/", a, b, ct.promote(a.type, b.type)) + return _pyd("Div", [a, b], res) + + if op == "//": + _check_numeric(op, a, b, fail) + return _pyd("FloorDiv", [a, b], ct.promote(a.type, b.type)) + + if op == "%": + _check_numeric(op, a, b, fail) + t = ct.promote(a.type, b.type) + # For unsigned integers C++ '%' already agrees with Python's. + if a.type.is_integral and b.type.is_integral and not a.type.is_signed and not b.type.is_signed: + return _binary("%", a, b, t) + return _pyd("Mod", [a, b], t) + + if op == "**": + _check_numeric(op, a, b, fail) + t = ct.promote(a.type, b.type) + if t.is_integral: + return _pyd("Pow", [a, b], t) + return _pyd("Pow", [a, b], ct.with_scalar(t, DOUBLE_T)) + + if op in ("|", "&", "^"): + # Python/numpy overload these for element-wise logic on booleans. + if _both_boolish(a, b): + cpp = {"|": "||", "&": "&&", "^": "!="}[op] + return _binary(cpp, a, b, ct.bool_like(ct.promote(a.type, b.type))) + if a.type.is_bool != b.type.is_bool and not (a.type.is_unknown or b.type.is_unknown): + fail( + "Mixing a boolean and a non-boolean operand in '{}' is ambiguous".format(op), + hint="'|' is a logical or between booleans but a bitwise or between integers. " + "Make both operands booleans, or cast explicitly.", + ) + if not (a.type.is_integral or a.type.is_unknown) or not (b.type.is_integral or b.type.is_unknown): + fail("Operator '{}' requires integer or boolean operands".format(op)) + return _binary(op, a, b, ct.promote(a.type, b.type)) + + if op in ("<<", ">>"): + if not (a.type.is_integral or a.type.is_unknown) or not (b.type.is_integral or b.type.is_unknown): + fail("Operator '{}' requires integer operands".format(op)) + return _binary(op, a, b, a.type) + + if op == "@": + fail("The matrix multiplication operator '@' is not supported") + + fail("Binary operator '{}' is not supported".format(op)) + + +def unaryop(op, a, fail): + """Lower a Python unary operator to C++.""" + if op == "+": + return Value("+{}".format(a.paren(PREC_UNARY)), a.type, PREC_UNARY) + if op == "-": + if a.type.kind == ct.OPAQUE: + fail("Unary '-' is not supported for values of C++ type '{}'".format(a.type.name)) + t = a.type + if t.is_bool: + t = ct.with_scalar(t, ct.INT_T) + return Value("-{}".format(a.paren(PREC_UNARY)), t, PREC_UNARY) + if op == "not": + return Value("!{}".format(truth(a, fail).paren(PREC_UNARY)), ct.bool_like(a.type), PREC_UNARY) + if op == "~": + if a.type.is_bool: + if a.type.is_container: + # numpy's '~' on a boolean array is a logical not. + return Value("!{}".format(a.paren(PREC_UNARY)), a.type, PREC_UNARY) + fail( + "'~' on a boolean scalar is not supported", + hint="In Python '~True' is -2, while on a numpy boolean array it is a logical " + "not. Use 'not x' for scalars.", + ) + if not (a.type.is_integral or a.type.is_unknown): + fail("Operator '~' requires an integer operand") + return Value("~{}".format(a.paren(PREC_UNARY)), a.type, PREC_UNARY) + fail("Unary operator '{}' is not supported".format(op)) + + +def truth(a, fail): + """The C++ expression testing a value for Python truthiness.""" + if a.type.is_container: + fail( + "The truth value of an array is ambiguous", + hint="Use .any() or .all() to reduce the array to a single boolean.", + ) + if a.type.is_bool or a.type.is_unknown: + return a + if a.type.is_numeric: + zero = Value("0", ct.INT_T) + return _binary("!=", a, zero, BOOL_T) + fail("Values of C++ type '{}' cannot be used as a condition".format(a.type)) + + +def compare(op, a, b, fail): + """Lower a single Python comparison to C++.""" + if op in ("is", "is not", "in", "not in"): + fail( + "The '{}' operator is not supported".format(op), + hint="Only the arithmetic comparisons ==, !=, <, <=, > and >= can be translated.", + ) + if a.type.kind == ct.OPAQUE or b.type.kind == ct.OPAQUE: + # Let C++ resolve the operator on the class; we cannot infer the result. + return _binary(op, a, b, UNKNOWN_T) + return _binary(op, a, b, ct.bool_like(ct.promote(a.type, b.type))) + + +def boolop(op, values, fail): + """Lower 'and'/'or'. Only defined for scalars, as in numpy.""" + cpp = "&&" if op == "and" else "||" + for v in values: + if v.type.is_container: + fail( + "'{}' is not defined for arrays".format(op), + hint="Use '&' and '|' for element-wise logic, as you would with numpy.", + ) + out = truth(values[0], fail) + for v in values[1:]: + out = _binary(cpp, out, truth(v, fail), BOOL_T) + return out + + +def ternary(cond, a, b, fail): + """Lower a Python conditional expression.""" + if cond.type.is_container: + return _pyd("Where", [cond, a, b], ct.with_scalar(cond.type, ct.promote(a.type, b.type).scalar())) + t = ct.promote(a.type, b.type) + code = "{} ? {} : {}".format(truth(cond, fail).paren(PREC_COND + 1), a.paren(PREC_COND + 1), b.paren(PREC_COND)) + return Value(code, t, PREC_COND) + + +# --------------------------------------------------------------------------- +# Indexing +# --------------------------------------------------------------------------- + + +def subscript(obj, index, fail): + """Lower v[i] for an integer index or a boolean mask.""" + if obj.type.kind == ct.OPAQUE or obj.type.is_unknown: + return Value("{}[{}]".format(obj.paren(PREC_PRIMARY), index.code), UNKNOWN_T, PREC_PRIMARY) + if not obj.type.is_container: + fail("Cannot index a value of type '{}'".format(obj.type)) + + elem = obj.type.scalar() + + if index.type.is_bool and index.type.is_container: + # RVec supports masking natively. + return Value("{}[{}]".format(obj.paren(PREC_PRIMARY), index.code), ct.rvec_of(elem), PREC_PRIMARY) + + if index.type.is_container: + fail( + "Indexing with an array of indices is not supported", + hint="Use ROOT::VecOps::Take through a boolean mask instead.", + ) + + if not (index.type.is_integral or index.type.is_unknown): + fail("Array indices must be integers, got '{}'".format(index.type)) + + # A non-negative literal, or an unsigned index, needs no wrap-around check. + literal_nonneg = index.code.isdigit() + if literal_nonneg or (index.type.is_integral and not index.type.is_signed): + return Value("{}[{}]".format(obj.paren(PREC_PRIMARY), index.code), elem, PREC_PRIMARY) + return _pyd("Index", [obj, index], elem) + + +def slice_(obj, start, stop, step, fail): + """Lower v[a:b:c] with full Python slice semantics.""" + if not obj.type.is_container: + if obj.type.kind == ct.OPAQUE: + fail("Cannot slice a value of C++ type '{}'".format(obj.type.name)) + fail("Cannot slice a value of type '{}'".format(obj.type)) + + elem = obj.type.scalar() + out_t = ct.rvec_of(elem) + + # v[::-1] is a plain reverse; emit the readable form. + if start is None and stop is None and step is not None and step.code == "-1": + return call("ROOT::VecOps::Reverse", [obj], out_t) + if start is None and stop is None and step is None: + return Value(obj.code, out_t, obj.prec) + + zero = Value("0", LONG_T) + one = Value("1", LONG_T) + args = [ + obj, + start if start is not None else zero, + stop if stop is not None else zero, + step if step is not None else one, + Value("true" if start is not None else "false", BOOL_T), + Value("true" if stop is not None else "false", BOOL_T), + ] + return _pyd("Slice", args, out_t) + + +# --------------------------------------------------------------------------- +# Methods and functions +# --------------------------------------------------------------------------- + +#: numpy-style reductions available as methods on arrays. +ARRAY_METHODS = { + "sum": ("Sum", "scalar"), + "prod": ("Prod", "scalar"), + "mean": ("Mean", "double"), + "std": ("Std", "double"), + "min": ("Min", "scalar"), + "max": ("Max", "scalar"), + "argmin": ("ArgMin", "long"), + "argmax": ("ArgMax", "long"), + "any": ("Any", "bool"), + "all": ("All", "bool"), +} + +#: numpy / math functions, mapped to the C++ support library. +UNARY_FUNCTIONS = { + "abs": "Abs", + "absolute": "Abs", + "fabs": "Abs", + "sqrt": "Sqrt", + "cbrt": "Cbrt", + "exp": "Exp", + "exp2": "Exp2", + "expm1": "Expm1", + "log": "Log", + "log2": "Log2", + "log10": "Log10", + "log1p": "Log1p", + "sin": "Sin", + "cos": "Cos", + "tan": "Tan", + "arcsin": "Asin", + "asin": "Asin", + "arccos": "Acos", + "acos": "Acos", + "arctan": "Atan", + "atan": "Atan", + "sinh": "Sinh", + "cosh": "Cosh", + "tanh": "Tanh", + "arcsinh": "Asinh", + "asinh": "Asinh", + "arccosh": "Acosh", + "acosh": "Acosh", + "arctanh": "Atanh", + "atanh": "Atanh", + "floor": "Floor", + "ceil": "Ceil", + "trunc": "Trunc", + "rint": "Round", + "round": "Round", + "erf": "Erf", + "erfc": "Erfc", + "lgamma": "Lgamma", + "gamma": "Tgamma", + "tgamma": "Tgamma", +} + +#: Functions whose result is always a double, whatever the argument type. +_DOUBLE_RESULT = frozenset( + [ + "Sqrt", + "Cbrt", + "Exp", + "Exp2", + "Expm1", + "Log", + "Log2", + "Log10", + "Log1p", + "Sin", + "Cos", + "Tan", + "Asin", + "Acos", + "Atan", + "Sinh", + "Cosh", + "Tanh", + "Asinh", + "Acosh", + "Atanh", + "Erf", + "Erfc", + "Lgamma", + "Tgamma", + "Atan2", + "Hypot", + "Fmod", + ] +) + +BINARY_FUNCTIONS = { + "arctan2": "Atan2", + "atan2": "Atan2", + "hypot": "Hypot", + "fmod": "Fmod", + "maximum": "Maximum", + "minimum": "Minimum", + "power": "Pow", + "pow": "Pow", +} + +#: numpy reductions that also exist as free functions. +FREE_REDUCTIONS = { + "sum": "Sum", + "prod": "Prod", + "mean": "Mean", + "average": "Mean", + "std": "Std", + "amin": "Min", + "amax": "Max", + "argmin": "ArgMin", + "argmax": "ArgMax", + "any": "Any", + "all": "All", +} + +#: numpy constants. +CONSTANTS = { + "pi": ("M_PI", DOUBLE_T), + "e": ("M_E", DOUBLE_T), + "inf": ("std::numeric_limits::infinity()", DOUBLE_T), + "nan": ("std::numeric_limits::quiet_NaN()", DOUBLE_T), + "tau": ("(2. * M_PI)", DOUBLE_T), +} + + +def _reduction_type(kind, obj): + if kind == "double": + return DOUBLE_T + if kind == "long": + return LONG_T + if kind == "bool": + return BOOL_T + elem = obj.type.scalar() + if elem.is_bool: + return LONG_T + return elem + + +def array_method(obj, name, args, fail): + """Lower a numpy-style method call on an array.""" + fn, kind = ARRAY_METHODS[name] + if args: + fail( + "'{}' does not accept arguments here".format(name), + hint="Reductions over an axis are not supported; the arrays are one dimensional.", + ) + return _pyd(fn, [obj], _reduction_type(kind, obj)) + + +def unary_function(fn, arg, fail): + """Lower a unary numpy/math function.""" + result = ct.with_scalar(arg.type, DOUBLE_T) if fn in _DOUBLE_RESULT else arg.type + return _pyd(fn, [arg], result) + + +def binary_function(fn, a, b, fail): + """Lower a binary numpy/math function.""" + t = ct.promote(a.type, b.type) + if fn in _DOUBLE_RESULT: + t = ct.with_scalar(t, DOUBLE_T) + return _pyd(fn, [a, b], t) + + +def cast(target, arg, fail): + """Lower float(x) / int(x) / bool(x).""" + if arg.type.is_container: + fail( + "Cannot convert an array with '{}()'".format(target), + hint="Use .astype(...) on a numpy array, or reduce the array first.", + ) + if target == "float": + return Value("static_cast({})".format(arg.code), DOUBLE_T, PREC_UNARY) + if target == "int": + # Python's int() truncates towards zero, and so does a C++ cast. + return Value("static_cast({})".format(arg.code), LONG_T, PREC_UNARY) + if target == "bool": + return truth(arg, fail) + fail("Conversion to '{}' is not supported".format(target)) + + +def length(arg, fail): + """Lower len(x).""" + if not (arg.type.is_container or arg.type.is_unknown or arg.type.kind == ct.OPAQUE): + fail("len() is only defined for arrays, got '{}'".format(arg.type)) + return _pyd("Len", [arg], LONG_T) + + +def cpp_method(obj, name, args): + """Pass a method call through to C++ untouched. + + This is what makes arbitrary ROOT classes usable: the transpiler does not + need to know anything about PtEtaPhiMVector to translate 'v.M()'. + """ + return Value( + "{}.{}({})".format(obj.paren(PREC_PRIMARY), name, ", ".join(a.code for a in args)), + UNKNOWN_T, + PREC_PRIMARY, + ) + + +def cpp_member(obj, name): + """Pass a data member access through to C++ untouched.""" + return Value("{}.{}".format(obj.paren(PREC_PRIMARY), name), UNKNOWN_T, PREC_PRIMARY) + + +def dtype_to_cpp(obj, fail): + """Map a numpy dtype, scalar type or C++ type name to a C++ type name.""" + if isinstance(obj, str): + parsed = ct.parse_type(obj) + if parsed.kind == ct.FUND: + return parsed.name + cpp = ct.NUMPY_TO_CPP.get(obj) + if cpp is not None: + return cpp + fail("'{}' is not a supported element type".format(obj)) + try: + import numpy as np + + name = np.dtype(obj).name + except Exception: + fail("Cannot interpret {!r} as an element type".format(obj)) + cpp = ct.NUMPY_TO_CPP.get(name) + if cpp is None: + fail("The element type '{}' has no supported C++ counterpart".format(name)) + return cpp + + +def array_from_elements(values, fail): + """Build an RVec out of individually computed elements, as np.array does.""" + if not values: + fail("An empty array cannot be built: its element type would be unknown") + elem = values[0].type + for v in values[1:]: + elem = ct.promote(elem, v.type) + if elem.is_container: + fail("Nested arrays are not supported; only one dimensional arrays can be built") + if elem.is_unknown: + fail( + "Cannot determine the element type of the array", + hint="Give the elements a known type, or pass dtype=... via .astype().", + ) + type_ = ct.rvec_of(elem) + return Value("{}{{{}}}".format(type_.cpp(), ", ".join(v.code for v in values)), type_, PREC_PRIMARY) + + +def astype(obj, dtype_obj, fail): + """Lower x.astype(dtype).""" + cpp = dtype_to_cpp(dtype_obj, fail) + if not obj.type.is_container: + if obj.type.is_unknown or obj.type.is_numeric: + return Value("static_cast<{}>({})".format(cpp, obj.code), ct.fund(cpp), PREC_UNARY) + fail("astype() is not defined for values of type '{}'".format(obj.type)) + target = ct.rvec_of(ct.fund(cpp)) + if obj.type == target: + return obj + return Value( + "{}::Return<{}>({})".format(PYD, target.cpp(), obj.code), + target, + PREC_PRIMARY, + ) + + +def check_no_kwargs(kwargs, what, fail): + if kwargs: + fail("Keyword arguments are not supported in a call to {}".format(what)) + + +def unsupported(msg, hint=None): + raise PyDeclareError(msg if hint is None else "{}\n {}".format(msg, hint)) + + +def constant_value(obj, fail): + """Convert a Python object captured from the enclosing scope to a C++ literal.""" + if isinstance(obj, bool): + return literal(obj, fail) + if isinstance(obj, (int, float)): + return literal(obj, fail) + + try: + import numpy as np + except ImportError: + np = None + + if np is not None: + if isinstance(obj, np.generic): + cpp = ct.NUMPY_TO_CPP.get(obj.dtype.name) + if cpp is None: + return None + value = literal(obj.item(), fail) + return Value("static_cast<{}>({})".format(cpp, value.code), ct.fund(cpp), PREC_UNARY) + if isinstance(obj, np.ndarray): + if obj.ndim != 1: + return None + cpp = ct.NUMPY_TO_CPP.get(obj.dtype.name) + if cpp is None: + return None + return sequence_literal([x.item() for x in obj], ct.fund(cpp), fail) + + if ( + isinstance(obj, (list, tuple)) + and obj + and all(isinstance(x, (int, float)) and not isinstance(x, bool) for x in obj) + ): + elem = DOUBLE_T if any(isinstance(x, float) for x in obj) else LONG_T + return sequence_literal(list(obj), elem, fail) + if isinstance(obj, (list, tuple)) and obj and all(isinstance(x, bool) for x in obj): + return sequence_literal(list(obj), BOOL_T, fail) + + return None diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py new file mode 100644 index 0000000000000..580a02fab76b1 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py @@ -0,0 +1,65 @@ +# Author: Jonas Rembser CERN 08/2026 + +################################################################################ +# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +"""Errors raised by the Python-to-C++ transpiler backends. + +The guiding principle of this package is that anything outside the supported +subset must fail *loudly*, at declaration time, with a message that points at +the offending line of Python source. Silently approximating Python semantics +in C++ would mean silently wrong physics results, which is far worse than an +error. +""" + +import textwrap + + +class PyDeclareError(Exception): + """Raised when a Python callable cannot be translated to C++. + + The message carries the location in the user's Python source whenever the + backend was able to determine it. + """ + + +def _source_context(func, lineno, col_offset=None): + """Return a ' File "...", line N' block quoting the offending source line.""" + import inspect + + try: + lines, first = inspect.getsourcelines(func) + filename = inspect.getsourcefile(func) or "" + except (OSError, TypeError): + return "" + + # inspect gives 1-based file line numbers; AST line numbers are relative to + # the start of the (dedented) function source. + if lineno is None: + return ' File "{}", line {}\n'.format(filename, first) + + abs_lineno = first + lineno - 1 + idx = lineno - 1 + text = lines[idx].rstrip("\n") if 0 <= idx < len(lines) else "" + stripped = text.strip() + out = ' File "{}", line {}\n {}\n'.format(filename, abs_lineno, stripped) + if col_offset is not None and text: + caret_col = col_offset - (len(text) - len(text.lstrip())) + if caret_col >= 0: + out += " " + " " * caret_col + "^\n" + return out + + +def raise_unsupported(func, node, what, hint=None): + """Raise a PyDeclareError for an unsupported construct at an AST node.""" + lineno = getattr(node, "lineno", None) + col = getattr(node, "col_offset", None) + msg = "{}\n{}".format(what, _source_context(func, lineno, col)) + if hint: + msg += textwrap.indent("\n" + hint.strip() + "\n", " ") + raise PyDeclareError(msg.rstrip() + "\n") diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py new file mode 100644 index 0000000000000..1e2a9a3aa2775 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py @@ -0,0 +1,692 @@ +# Author: Jonas Rembser CERN 08/2026 + +################################################################################ +# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +"""The C++ runtime support library used by the generated code. + +Everything in here exists for one reason: Python's operator semantics are not +C++'s. Rather than relying on the transpiler's type inference to be complete +enough to pick the right C++ spelling, the semantics are pushed into small +templates that get it right for whatever type the compiler actually deduces. +Inference then only has to be good enough to choose *which* helper to call. + +The header is declared to cling exactly once per process, lazily. +""" + +_DECLARED = False + +CPP_SUPPORT = r""" +#ifndef ROOT_PYDECLARE_SUPPORT +#define ROOT_PYDECLARE_SUPPORT + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ROOT { +namespace Internal { +namespace PyDeclare { + +// --------------------------------------------------------------------------- +// Traits +// --------------------------------------------------------------------------- + +template +struct IsRVec : std::false_type {}; +template +struct IsRVec> : std::true_type {}; + +template +struct IsStdVector : std::false_type {}; +template +struct IsStdVector> : std::true_type {}; + +template +struct IsStdArray : std::false_type {}; +template +struct IsStdArray> : std::true_type {}; + +template +struct IsVecLike : std::integral_constant::value || IsStdVector::value || IsStdArray::value> {}; + +/// The element type of a container, or the type itself for scalars. +template ::type>::value> +struct ElemTypeImpl { + using type = typename std::decay::type::value_type; +}; +template +struct ElemTypeImpl { + using type = typename std::decay::type; +}; +template +using ElemType = typename ElemTypeImpl::type; + +/// Negativity test that does not warn for unsigned types. +template +constexpr bool IsNeg(T x) +{ + if constexpr (std::is_signed::value) { + return x < T(0); + } else { + (void)x; + return false; + } +} + +// --------------------------------------------------------------------------- +// Adapting the supported container types to RVec +// +// RVec is the only container with the full set of element-wise operators, so +// std::vector and std::array arguments are viewed as RVecs on entry. The +// adopting RVec constructor makes this a non-owning view: no copy per event. +// --------------------------------------------------------------------------- + +template +const ROOT::VecOps::RVec &AsRVec(const ROOT::VecOps::RVec &v) +{ + return v; +} + +template +ROOT::VecOps::RVec AsRVec(const std::vector &v) +{ + return ROOT::VecOps::RVec(const_cast(v.data()), v.size()); +} + +template +ROOT::VecOps::RVec AsRVec(const std::vector &v) +{ + // std::vector is a bitfield and has no data(); this one has to copy. + return ROOT::VecOps::RVec(v.begin(), v.end()); +} + +template +ROOT::VecOps::RVec AsRVec(const std::array &v) +{ + return ROOT::VecOps::RVec(const_cast(v.data()), N); +} + +// --------------------------------------------------------------------------- +// Element-wise application over a mix of containers and scalars +// --------------------------------------------------------------------------- + +template +std::size_t VecSizeOf(const T &x) +{ + if constexpr (IsVecLike::type>::value) { + return x.size(); + } else { + (void)x; + return 0u; + } +} + +template +decltype(auto) ElemAt(const T &x, std::size_t i) +{ + if constexpr (IsVecLike::type>::value) { + return x[i]; + } else { + (void)i; + return x; + } +} + +template +auto ZipMap(const A &a, const B &b, F f) +{ + constexpr bool aVec = IsVecLike::type>::value; + constexpr bool bVec = IsVecLike::type>::value; + static_assert(aVec || bVec, "ZipMap needs at least one container argument"); + const std::size_t na = VecSizeOf(a); + const std::size_t nb = VecSizeOf(b); + if (aVec && bVec && na != nb) { + throw std::runtime_error("PyDeclare: operands could not be broadcast together"); + } + const std::size_t n = aVec ? na : nb; + using R = typename std::decay::type; + ROOT::VecOps::RVec out(n); + for (std::size_t i = 0; i < n; ++i) { + out[i] = f(ElemAt(a, i), ElemAt(b, i)); + } + return out; +} + +// --------------------------------------------------------------------------- +// Python arithmetic semantics +// --------------------------------------------------------------------------- + +/// Python's '/' is always true division, even between integers. +template +auto AsDoubleVec(const T &x) +{ + using E = ElemType; + ROOT::VecOps::RVec out(x.size()); + for (std::size_t i = 0; i < x.size(); ++i) { + out[i] = static_cast(static_cast(x[i])); + } + return out; +} + +template +auto Div(const A &a, const B &b) +{ + using EA = ElemType; + using EB = ElemType; + if constexpr (std::is_floating_point::value || std::is_floating_point::value) { + // C++ already promotes to floating point here, no fixup needed. + return a / b; + } else if constexpr (IsVecLike::type>::value) { + if constexpr (IsVecLike::type>::value) { + return AsDoubleVec(a) / AsDoubleVec(b); + } else { + return AsDoubleVec(a) / static_cast(b); + } + } else if constexpr (IsVecLike::type>::value) { + return static_cast(a) / AsDoubleVec(b); + } else { + return static_cast(a) / static_cast(b); + } +} + +/// Python's '//' floors; C++ integer division truncates towards zero. +template +auto FloorDivScalar(A a, B b) +{ + if constexpr (std::is_integral::value && std::is_integral::value) { + auto q = a / b; + auto r = a % b; + if (r != 0 && (IsNeg(r) != IsNeg(b))) { + q -= 1; + } + return q; + } else { + return std::floor(static_cast(a) / static_cast(b)); + } +} + +template +auto FloorDiv(const A &a, const B &b) +{ + if constexpr (IsVecLike::type>::value || + IsVecLike::type>::value) { + return ZipMap(a, b, [](auto x, auto y) { return FloorDivScalar(x, y); }); + } else { + return FloorDivScalar(a, b); + } +} + +/// Python's '%' takes the sign of the divisor; C++'s takes the sign of the +/// dividend. -7 % 3 is 2 in Python and -1 in C++. +template +auto ModScalar(A a, B b) +{ + if constexpr (std::is_integral::value && std::is_integral::value) { + auto r = a % b; + if (r != 0 && (IsNeg(r) != IsNeg(b))) { + r += b; + } + return r; + } else { + const double bd = static_cast(b); + double r = std::fmod(static_cast(a), bd); + if (r != 0.0 && ((r < 0.0) != (bd < 0.0))) { + r += bd; + } + return r; + } +} + +template +auto Mod(const A &a, const B &b) +{ + if constexpr (IsVecLike::type>::value || + IsVecLike::type>::value) { + return ZipMap(a, b, [](auto x, auto y) { return ModScalar(x, y); }); + } else { + return ModScalar(a, b); + } +} + +/// Python's '**'. int ** non-negative int stays integral, as in Python. +template +auto PowScalar(A a, B b) +{ + if constexpr (std::is_integral::value && std::is_integral::value) { + if (IsNeg(b)) { + throw std::runtime_error("PyDeclare: integer ** negative integer would return a float in Python; " + "cast a base or exponent to a floating point type"); + } + A result = A(1); + A base = a; + auto e = b; + while (e > 0) { + if (e & 1) { + result *= base; + } + e >>= 1; + if (e) { + base *= base; + } + } + return result; + } else { + return std::pow(a, b); + } +} + +template +auto Pow(const A &a, const B &b) +{ + if constexpr (IsVecLike::type>::value || + IsVecLike::type>::value) { + return ZipMap(a, b, [](auto x, auto y) { return PowScalar(x, y); }); + } else { + return PowScalar(a, b); + } +} + +// --------------------------------------------------------------------------- +// Indexing and slicing with Python semantics +// --------------------------------------------------------------------------- + +template +decltype(auto) Index(const V &v, I i) +{ + if constexpr (std::is_signed::value) { + if (i < 0) { + const long long n = static_cast(v.size()); + const long long j = n + static_cast(i); + if (j < 0) { + throw std::out_of_range("PyDeclare: index out of range"); + } + return v[static_cast(j)]; + } + } + return v[static_cast(i)]; +} + +template +decltype(auto) Index(V &v, I i) +{ + if constexpr (std::is_signed::value) { + if (i < 0) { + const long long n = static_cast(v.size()); + const long long j = n + static_cast(i); + if (j < 0) { + throw std::out_of_range("PyDeclare: index out of range"); + } + return v[static_cast(j)]; + } + } + return v[static_cast(i)]; +} + +/// Full Python slice semantics, including negative bounds and negative steps. +template +auto Slice(const V &v, long long start, long long stop, long long step, bool hasStart, bool hasStop) +{ + using E = ElemType; + if (step == 0) { + throw std::runtime_error("PyDeclare: slice step cannot be zero"); + } + const long long n = static_cast(v.size()); + long long b = 0; + long long e = 0; + if (step > 0) { + b = hasStart ? (start < 0 ? std::max(0LL, n + start) : std::min(start, n)) : 0; + e = hasStop ? (stop < 0 ? std::max(0LL, n + stop) : std::min(stop, n)) : n; + } else { + b = hasStart ? (start < 0 ? std::max(-1LL, n + start) : std::min(start, n - 1)) : n - 1; + e = hasStop ? (stop < 0 ? std::max(-1LL, n + stop) : std::min(stop, n - 1)) : -1; + } + ROOT::VecOps::RVec out; + if (step > 0) { + if (e > b) { + out.reserve(static_cast((e - b + step - 1) / step)); + } + for (long long i = b; i < e; i += step) { + out.push_back(v[static_cast(i)]); + } + } else { + if (b > e) { + out.reserve(static_cast((b - e - step - 1) / (-step))); + } + for (long long i = b; i > e; i += step) { + out.push_back(v[static_cast(i)]); + } + } + return out; +} + +template +long Len(const V &v) +{ + return static_cast(v.size()); +} + +// --------------------------------------------------------------------------- +// Reductions, with numpy's result conventions +// --------------------------------------------------------------------------- + +template +auto Sum(const V &v) +{ + using E = ElemType; + using R = typename std::conditional::value, long, E>::type; + R acc = R(0); + for (std::size_t i = 0; i < v.size(); ++i) { + acc += static_cast(v[i]); + } + return acc; +} + +template +auto Prod(const V &v) +{ + using E = ElemType; + using R = typename std::conditional::value, long, E>::type; + R acc = R(1); + for (std::size_t i = 0; i < v.size(); ++i) { + acc *= static_cast(v[i]); + } + return acc; +} + +template +double Mean(const V &v) +{ + if (v.empty()) { + return std::numeric_limits::quiet_NaN(); + } + double acc = 0.; + for (std::size_t i = 0; i < v.size(); ++i) { + acc += static_cast(v[i]); + } + return acc / static_cast(v.size()); +} + +/// numpy's std(), i.e. the population standard deviation (ddof = 0). Note +/// that ROOT::VecOps::StdDev is the *sample* standard deviation (ddof = 1). +template +double Std(const V &v) +{ + if (v.empty()) { + return std::numeric_limits::quiet_NaN(); + } + const double m = Mean(v); + double acc = 0.; + for (std::size_t i = 0; i < v.size(); ++i) { + const double d = static_cast(v[i]) - m; + acc += d * d; + } + return std::sqrt(acc / static_cast(v.size())); +} + +template +auto Min(const V &v) +{ + using E = ElemType; + if (v.empty()) { + throw std::runtime_error("PyDeclare: min() of an empty sequence"); + } + E best = v[0]; + for (std::size_t i = 1; i < v.size(); ++i) { + if (v[i] < best) { + best = v[i]; + } + } + return best; +} + +template +auto Max(const V &v) +{ + using E = ElemType; + if (v.empty()) { + throw std::runtime_error("PyDeclare: max() of an empty sequence"); + } + E best = v[0]; + for (std::size_t i = 1; i < v.size(); ++i) { + if (best < v[i]) { + best = v[i]; + } + } + return best; +} + +template +long ArgMin(const V &v) +{ + if (v.empty()) { + throw std::runtime_error("PyDeclare: argmin() of an empty sequence"); + } + std::size_t best = 0; + for (std::size_t i = 1; i < v.size(); ++i) { + if (v[i] < v[best]) { + best = i; + } + } + return static_cast(best); +} + +template +long ArgMax(const V &v) +{ + if (v.empty()) { + throw std::runtime_error("PyDeclare: argmax() of an empty sequence"); + } + std::size_t best = 0; + for (std::size_t i = 1; i < v.size(); ++i) { + if (v[best] < v[i]) { + best = i; + } + } + return static_cast(best); +} + +template +bool Any(const V &v) +{ + for (std::size_t i = 0; i < v.size(); ++i) { + if (v[i]) { + return true; + } + } + return false; +} + +template +bool All(const V &v) +{ + for (std::size_t i = 0; i < v.size(); ++i) { + if (!v[i]) { + return false; + } + } + return true; +} + +// --------------------------------------------------------------------------- +// Element-wise maths. Unqualified lookup picks std::f for scalars and +// ROOT::VecOps::f for RVecs. +// --------------------------------------------------------------------------- + +#define PYDECLARE_UNARY_MATH(NAME, FN) \ + template \ + auto NAME(const T &x) \ + { \ + using std::FN; \ + using namespace ROOT::VecOps; \ + return FN(x); \ + } + +PYDECLARE_UNARY_MATH(Abs, abs) +PYDECLARE_UNARY_MATH(Sqrt, sqrt) +PYDECLARE_UNARY_MATH(Cbrt, cbrt) +PYDECLARE_UNARY_MATH(Exp, exp) +PYDECLARE_UNARY_MATH(Exp2, exp2) +PYDECLARE_UNARY_MATH(Expm1, expm1) +PYDECLARE_UNARY_MATH(Log, log) +PYDECLARE_UNARY_MATH(Log2, log2) +PYDECLARE_UNARY_MATH(Log10, log10) +PYDECLARE_UNARY_MATH(Log1p, log1p) +PYDECLARE_UNARY_MATH(Sin, sin) +PYDECLARE_UNARY_MATH(Cos, cos) +PYDECLARE_UNARY_MATH(Tan, tan) +PYDECLARE_UNARY_MATH(Asin, asin) +PYDECLARE_UNARY_MATH(Acos, acos) +PYDECLARE_UNARY_MATH(Atan, atan) +PYDECLARE_UNARY_MATH(Sinh, sinh) +PYDECLARE_UNARY_MATH(Cosh, cosh) +PYDECLARE_UNARY_MATH(Tanh, tanh) +PYDECLARE_UNARY_MATH(Asinh, asinh) +PYDECLARE_UNARY_MATH(Acosh, acosh) +PYDECLARE_UNARY_MATH(Atanh, atanh) +PYDECLARE_UNARY_MATH(Floor, floor) +PYDECLARE_UNARY_MATH(Ceil, ceil) +PYDECLARE_UNARY_MATH(Trunc, trunc) +PYDECLARE_UNARY_MATH(Erf, erf) +PYDECLARE_UNARY_MATH(Erfc, erfc) +PYDECLARE_UNARY_MATH(Lgamma, lgamma) +PYDECLARE_UNARY_MATH(Tgamma, tgamma) + +#undef PYDECLARE_UNARY_MATH + +/// Python's round() and numpy's round() both round halves to even, while +/// std::round rounds halves away from zero. std::nearbyint under the default +/// rounding mode does what Python does. +inline double RoundScalar(double x) +{ + const double r = std::nearbyint(x); + return r == 0. ? 0. : r; // normalise -0. +} + +template +auto Round(const T &x) +{ + if constexpr (IsVecLike::type>::value) { + ROOT::VecOps::RVec out(x.size()); + for (std::size_t i = 0; i < x.size(); ++i) { + out[i] = RoundScalar(static_cast(x[i])); + } + return out; + } else { + return RoundScalar(static_cast(x)); + } +} + +#define PYDECLARE_BINARY_MATH(NAME, FN) \ + template \ + auto NAME(const A &a, const B &b) \ + { \ + using std::FN; \ + using namespace ROOT::VecOps; \ + return FN(a, b); \ + } + +PYDECLARE_BINARY_MATH(Atan2, atan2) +PYDECLARE_BINARY_MATH(Hypot, hypot) +PYDECLARE_BINARY_MATH(Fmod, fmod) + +#undef PYDECLARE_BINARY_MATH + +template +auto Maximum(const A &a, const B &b) +{ + if constexpr (IsVecLike::type>::value || + IsVecLike::type>::value) { + return ZipMap(a, b, [](auto x, auto y) { return x > y ? x : y; }); + } else { + return a > b ? a : b; + } +} + +template +auto Minimum(const A &a, const B &b) +{ + if constexpr (IsVecLike::type>::value || + IsVecLike::type>::value) { + return ZipMap(a, b, [](auto x, auto y) { return x < y ? x : y; }); + } else { + return a < b ? a : b; + } +} + +template +auto Where(const C &c, const A &a, const B &b) +{ + if constexpr (IsVecLike::type>::value) { + const std::size_t n = c.size(); + using R = typename std::decay::type; + ROOT::VecOps::RVec out(n); + for (std::size_t i = 0; i < n; ++i) { + out[i] = c[i] ? static_cast(ElemAt(a, i)) : static_cast(ElemAt(b, i)); + } + return out; + } else { + return c ? a : b; + } +} + +// --------------------------------------------------------------------------- +// Returning +// --------------------------------------------------------------------------- + +/// Convert the value of the translated return expression to the declared +/// return type. Anything that cannot be converted is a compile error, which +/// is exactly the loud failure we want. +template +Out Return(T &&x) +{ + using D = typename std::decay::type; + if constexpr (std::is_same::value) { + return std::forward(x); + } else if constexpr (IsStdArray::value && IsVecLike::value) { + Out out{}; + if (x.size() != out.size()) { + throw std::runtime_error("PyDeclare: returned sequence has the wrong length for the declared std::array"); + } + std::copy(x.begin(), x.end(), out.begin()); + return out; + } else if constexpr (IsVecLike::value && IsVecLike::value) { + return Out(x.begin(), x.end()); + } else { + return static_cast(x); + } +} + +} // namespace PyDeclare +} // namespace Internal +} // namespace ROOT + +#endif // ROOT_PYDECLARE_SUPPORT +""" + + +def ensure_declared(): + """Declare the C++ support library to cling, once per process.""" + global _DECLARED + if _DECLARED: + return + import ROOT + + # RVec entities are not in the global module index, and gInterpreter.Declare + # is not allowed to autoload libROOTVecOps, so preload it here. + ROOT.gSystem.Load("libROOTVecOps") + if not ROOT.gInterpreter.Declare(CPP_SUPPORT): + raise RuntimeError("PyDeclare: failed to declare the C++ support library to cling") + _DECLARED = True diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py new file mode 100644 index 0000000000000..50b93398de511 --- /dev/null +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py @@ -0,0 +1,652 @@ +# Author: Jonas Rembser CERN 08/2026 + +################################################################################ +# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +"""The tracing backend. + +Instead of reading the Python source, this backend *runs* the callable once +with symbolic arguments. Every operator, method and numpy ufunc applied to a +symbol records itself, and what comes back is a DAG describing the computation, +which is then emitted as C++. + +The trade-off against the AST backend is control flow. A traced symbol has no +value, so `if x > 0:` cannot pick a branch; `__bool__` raises instead of +guessing. Python-level control flow that does *not* depend on the arguments is +a different matter -- it unrolls, which is occasionally exactly what you want -- +but since it silently changes what the generated code does relative to reading +the source, it has to be asked for explicitly with ``unroll=True``. +""" + +import ast +import inspect +import sys +import textwrap + +from . import emit +from .errors import PyDeclareError + +_CONTROL_FLOW_NODES = (ast.If, ast.For, ast.While, ast.Try, ast.With, ast.IfExp) + + +class _TraceError(PyDeclareError): + pass + + +def _here(func): + """Format the position inside the traced callable, for error messages.""" + import traceback + + code = getattr(func, "__code__", None) + frames = traceback.extract_stack() + for frame in reversed(frames): + if code is not None and frame.filename == code.co_filename and frame.name == code.co_name: + return '\n File "{}", line {}\n {}\n'.format(frame.filename, frame.lineno, (frame.line or "").strip()) + return "" + + +class Sym: + """A symbolic value standing in for one C++ expression. + + Nodes are built lazily: each carries a builder that turns the *reference + codes* of its children into a Value. That lets the emitter decide after + the fact whether a shared sub-expression should become a named temporary + rather than being pasted in twice. + """ + + __slots__ = ("ctx", "builder", "children", "value", "name", "uses") + + __array_priority__ = 1000.0 + + def __init__(self, ctx, builder, children, value): + self.ctx = ctx + self.builder = builder + self.children = children + self.value = value # eagerly built, for its type + self.name = None # set if this node is materialised as a temporary + self.uses = 0 + ctx.nodes.append(self) + + # -- construction helpers ---------------------------------------------- + + @property + def type(self): + return self.value.type + + def rebuild(self, child_values): + return self.builder(child_values) + + def __repr__(self): + return "Sym({!r}, {})".format(self.value.code, self.value.type) + + # -- the loud failures -------------------------------------------------- + + def _refuse(self, what, hint): + raise _TraceError( + "{} while tracing '{}'.{}\n {}".format( + what, getattr(self.ctx.func, "__name__", ""), _here(self.ctx.func), hint + ) + ) + + def __bool__(self): + self._refuse( + "A traced value was converted to a Python bool", + "This happens for 'if x', 'while x', 'not x', 'x and y', 'x or y' and 'assert x'. " + "The interpreter resolves all of them by asking the value for its truth, and a " + "symbol has none to give: the tracing backend records what one Python run does, so " + "it never sees the operation at all.\n" + " Use backend='ast', which reads the source and translates these directly. For " + "element-wise logic the numpy spelling also works in both backends: '~x' for not, " + "'&' for and, '|' for or, and np.where(cond, a, b) for a conditional.", + ) + + def __len__(self): + self._refuse( + "len() was called on a traced array", + "The length is only known at run time. Use x.size, which stays symbolic.", + ) + + def __iter__(self): + self._refuse( + "A traced array was iterated over", + "The number of elements is only known at run time. Use element-wise array " + "expressions, or backend='ast' with an explicit loop.", + ) + + def __index__(self): + self._refuse("A traced value was used as an index into a Python object", "Index the array itself instead.") + + __int__ = __index__ + + def __float__(self): + self._refuse( + "A traced value was converted to a Python float", + "Arithmetic stays symbolic; use np.* functions rather than math.* ones, which force " + "a conversion to a Python number.", + ) + + def __hash__(self): + return id(self) + + # -- operators ---------------------------------------------------------- + + def _binop(self, op, other, swap=False): + a, b = (self.ctx.wrap(other), self) if swap else (self, self.ctx.wrap(other)) + return self.ctx.node(lambda vals: emit.binop(op, vals[0], vals[1], self.ctx.fail), [a, b]) + + def _cmp(self, op, other): + a, b = self, self.ctx.wrap(other) + return self.ctx.node(lambda vals: emit.compare(op, vals[0], vals[1], self.ctx.fail), [a, b]) + + +def _install_operators(): + binary = { + "add": "+", + "sub": "-", + "mul": "*", + "truediv": "/", + "floordiv": "//", + "mod": "%", + "pow": "**", + "lshift": "<<", + "rshift": ">>", + "and": "&", + "or": "|", + "xor": "^", + } + for pyname, op in binary.items(): + + def make(op=op, swap=False): + def method(self, other): + return self._binop(op, other, swap) + + return method + + setattr(Sym, "__{}__".format(pyname), make(op, False)) + setattr(Sym, "__r{}__".format(pyname), make(op, True)) + + for pyname, op in {"lt": "<", "le": "<=", "gt": ">", "ge": ">=", "eq": "==", "ne": "!="}.items(): + + def make_cmp(op=op): + def method(self, other): + return self._cmp(op, other) + + return method + + setattr(Sym, "__{}__".format(pyname), make_cmp(op)) + + for pyname, op in {"neg": "-", "pos": "+", "invert": "~"}.items(): + + def make_un(op=op): + def method(self): + return self.ctx.node(lambda vals: emit.unaryop(op, vals[0], self.ctx.fail), [self]) + + return method + + setattr(Sym, "__{}__".format(pyname), make_un(op)) + + def _abs(self): + return self.ctx.node(lambda vals: emit.unary_function("Abs", vals[0], self.ctx.fail), [self]) + + Sym.__abs__ = _abs + + +_install_operators() + + +# --- indexing --------------------------------------------------------------- + + +def _getitem(self, key): + ctx = self.ctx + if isinstance(key, slice): + parts = [key.start, key.stop, key.step] + syms = [None if p is None else ctx.wrap(p) for p in parts] + children = [self] + [s for s in syms if s is not None] + + def build(vals): + it = iter(vals[1:]) + start, stop, step = (next(it) if s is not None else None for s in syms) + return emit.slice_(vals[0], start, stop, step, ctx.fail) + + return ctx.node(build, children) + index = ctx.wrap(key) + return ctx.node(lambda vals: emit.subscript(vals[0], vals[1], ctx.fail), [self, index]) + + +Sym.__getitem__ = _getitem + + +def _setitem(self, key, value): + self._refuse( + "A traced array was assigned into", + "The tracing backend builds a pure expression; it has nowhere to put a mutation. " + "Use backend='ast' if you need to write into an array.", + ) + + +Sym.__setitem__ = _setitem + + +# --- attributes ------------------------------------------------------------- + + +class _Method: + """A method looked up on a symbol but not yet called.""" + + def __init__(self, sym, name): + self.sym = sym + self.name = name + + def __call__(self, *args, **kwargs): + ctx = self.sym.ctx + if kwargs: + raise _TraceError( + "Keyword arguments are not supported in a call to '{}'.{}".format(self.name, _here(ctx.func)) + ) + if self.name == "astype": + if len(args) != 1: + raise _TraceError("astype() takes exactly one argument.{}".format(_here(ctx.func))) + dtype = args[0] + return ctx.node(lambda vals: emit.astype(vals[0], dtype, ctx.fail), [self.sym]) + if self.name in emit.ARRAY_METHODS and self.sym.type.is_container: + if args: + raise _TraceError("'{}' does not take arguments here.{}".format(self.name, _here(ctx.func))) + return ctx.node(lambda vals: emit.array_method(vals[0], self.name, [], ctx.fail), [self.sym]) + # Anything else is passed straight through to C++: this is how methods + # on ROOT classes such as PtEtaPhiMVector::M() are translated. + children = [self.sym] + [ctx.wrap(a) for a in args] + return ctx.node(lambda vals: emit.cpp_method(vals[0], self.name, vals[1:]), children) + + +def _getattr(self, name): + if name.startswith("__") or name in Sym.__slots__: + raise AttributeError(name) + ctx = self.ctx + if self.type.is_container: + if name == "size": + return ctx.node(lambda vals: emit.length(vals[0], ctx.fail), [self]) + if name in emit.ARRAY_METHODS: + return _Method(self, name) + if name in ("T", "shape", "ndim", "dtype", "flat", "real", "imag"): + raise _TraceError( + "Arrays have no '{}' in the supported subset; they are always one dimensional " + "arrays of numbers.{}".format(name, _here(ctx.func)) + ) + return _Method(self, name) + if self.type.is_fund: + raise _TraceError("'{}' has no attribute '{}'.{}".format(self.type, name, _here(ctx.func))) + # Opaque C++ object: could be a data member or a method. + return _MaybeMember(self, name) + + +class _MaybeMember(emit.Value): + """A member access on an opaque C++ object: a value, or a method to call.""" + + __slots__ = ("sym", "name", "_node") + + def __init__(self, sym, name): + ctx = sym.ctx + node = ctx.node(lambda vals: emit.cpp_member(vals[0], name), [sym]) + super().__init__(node.value.code, node.value.type, node.value.prec) + self.sym = sym + self.name = name + self._node = node + + def __call__(self, *args, **kwargs): + # It was a method after all; drop the member node and build a call. + ctx = self.sym.ctx + ctx.discard(self._node) + return _Method(self.sym, self.name)(*args, **kwargs) + + +Sym.__getattr__ = _getattr + + +# --- numpy interoperability ------------------------------------------------- + +_UFUNC_BINOPS = { + "add": "+", + "subtract": "-", + "multiply": "*", + "true_divide": "/", + "divide": "/", + "floor_divide": "//", + "remainder": "%", + "mod": "%", + "power": "**", + "float_power": "**", + "left_shift": "<<", + "right_shift": ">>", + "bitwise_and": "&", + "bitwise_or": "|", + "bitwise_xor": "^", + "logical_and": "&", + "logical_or": "|", + "logical_xor": "^", +} +_UFUNC_COMPARES = { + "less": "<", + "less_equal": "<=", + "greater": ">", + "greater_equal": ">=", + "equal": "==", + "not_equal": "!=", +} +_UFUNC_UNARY = {"negative": "-", "positive": "+", "invert": "~", "logical_not": "not"} + + +def _array_ufunc(self, ufunc, method, *inputs, **kwargs): + ctx = self.ctx + name = getattr(ufunc, "__name__", "") + if method != "__call__" or kwargs.get("out") is not None: + raise _TraceError("numpy's '{}' method of ufunc '{}' is not supported.{}".format(method, name, _here(ctx.func))) + args = [ctx.wrap(a) for a in inputs] + + if name in _UFUNC_UNARY and len(args) == 1: + op = _UFUNC_UNARY[name] + return ctx.node(lambda vals: emit.unaryop(op, vals[0], ctx.fail), args) + if name in emit.UNARY_FUNCTIONS and len(args) == 1: + fn = emit.UNARY_FUNCTIONS[name] + return ctx.node(lambda vals: emit.unary_function(fn, vals[0], ctx.fail), args) + if name in _UFUNC_BINOPS and len(args) == 2: + op = _UFUNC_BINOPS[name] + return ctx.node(lambda vals: emit.binop(op, vals[0], vals[1], ctx.fail), args) + if name in _UFUNC_COMPARES and len(args) == 2: + op = _UFUNC_COMPARES[name] + return ctx.node(lambda vals: emit.compare(op, vals[0], vals[1], ctx.fail), args) + if name in emit.BINARY_FUNCTIONS and len(args) == 2: + fn = emit.BINARY_FUNCTIONS[name] + return ctx.node(lambda vals: emit.binary_function(fn, vals[0], vals[1], ctx.fail), args) + raise _TraceError( + "The numpy function '{}' is not available in the supported subset.{}".format(name, _here(ctx.func)) + ) + + +def _array_function(self, func, types, args, kwargs): + ctx = self.ctx + name = getattr(func, "__name__", "") + if kwargs: + raise _TraceError("Keyword arguments to numpy's '{}' are not supported.{}".format(name, _here(ctx.func))) + wrapped = [ctx.wrap(a) for a in args] + if name == "where" and len(wrapped) == 3: + return ctx.node(lambda vals: emit.ternary(vals[0], vals[1], vals[2], ctx.fail), wrapped) + if name in emit.FREE_REDUCTIONS and len(wrapped) == 1: + method = _REDUCTION_METHOD[emit.FREE_REDUCTIONS[name]] + return ctx.node(lambda vals: emit.array_method(vals[0], method, [], ctx.fail), wrapped) + raise _TraceError( + "The numpy function '{}' is not available in the supported subset.{}".format(name, _here(ctx.func)) + ) + + +_REDUCTION_METHOD = {fn: name for name, (fn, _) in emit.ARRAY_METHODS.items()} + +Sym.__array_ufunc__ = _array_ufunc +Sym.__array_function__ = _array_function + + +# --- tracer-aware builtins -------------------------------------------------- +# +# float(x), len(x) and friends are resolved by the interpreter to a call that +# must return a real Python object: float() insists on a float, len() on an int. +# A symbol cannot satisfy them. They can, however, be shadowed: the traced +# callable is re-created with a __builtins__ mapping in which these names point +# at tracer-aware versions, so the ordinary Python spelling keeps working. +# +# The operations the interpreter routes through __bool__ -- 'not x', 'x and y', +# 'if x' -- have no such hook, and are reported instead. + + +def _traced_builtin(name): + import builtins + + original = getattr(builtins, name) + + def wrapper(*args): + syms = [a for a in args if isinstance(a, Sym)] + if not syms: + return original(*args) + ctx = syms[0].ctx + wrapped = [ctx.wrap(a) for a in args] + + def build(vals): + if name in ("float", "int", "bool"): + return emit.cast(name, vals[0], ctx.fail) + if name == "len": + return emit.length(vals[0], ctx.fail) + if name == "abs": + return emit.unary_function("Abs", vals[0], ctx.fail) + if name == "round": + return emit.unary_function("Round", vals[0], ctx.fail) + if name == "sum": + return emit.array_method(vals[0], "sum", [], ctx.fail) + if name == "pow": + return emit.binop("**", vals[0], vals[1], ctx.fail) + if name in ("min", "max"): + if len(vals) == 1: + return emit.array_method(vals[0], name, [], ctx.fail) + return emit.binary_function("Maximum" if name == "max" else "Minimum", vals[0], vals[1], ctx.fail) + ctx.fail("The builtin '{}' is not available in the supported subset".format(name)) + + if name in ("float", "int", "bool", "len", "abs", "round", "sum") and len(wrapped) != 1: + ctx.fail("{}() takes exactly one argument here".format(name)) + if name == "pow" and len(wrapped) != 2: + ctx.fail("pow() takes exactly two arguments here") + if name in ("min", "max") and len(wrapped) not in (1, 2): + ctx.fail("{}() takes one or two arguments here".format(name)) + return ctx.node(build, wrapped) + + wrapper.__name__ = name + return wrapper + + +class _NumpyProxy: + """Stands in for the numpy module while tracing. + + Only np.array needs intercepting: unlike the ufuncs and the functions + covered by __array_function__, it has no dispatch protocol, so a list + containing symbols would silently become an object array. + """ + + def __init__(self, module): + object.__setattr__(self, "_module", module) + + def __getattr__(self, name): + if name == "array": + return _traced_np_array + return getattr(object.__getattribute__(self, "_module"), name) + + +def _traced_np_array(obj, dtype=None): + import numpy as np + + values = list(obj) if isinstance(obj, (list, tuple)) else [] + syms = [v for v in values if isinstance(v, Sym)] + if not syms: + return np.array(obj) if dtype is None else np.array(obj, dtype=dtype) + ctx = syms[0].ctx + children = [ctx.wrap(v) for v in values] + + def build(vals): + built = emit.array_from_elements(vals, ctx.fail) + return built if dtype is None else emit.astype(built, dtype, ctx.fail) + + return ctx.node(build, children) + + +TRACED_BUILTINS = ("float", "int", "bool", "len", "abs", "round", "sum", "pow", "min", "max") + + +def _with_traced_builtins(func): + """A copy of func whose builtin lookups see the tracer-aware versions.""" + import builtins + import types + + if not hasattr(func, "__code__"): + return func + env = dict(vars(builtins)) + for name in TRACED_BUILTINS: + env[name] = _traced_builtin(name) + glob = dict(getattr(func, "__globals__", {}) or {}) + glob["__builtins__"] = env + numpy_module = sys.modules.get("numpy") + if numpy_module is not None: + for name, value in list(glob.items()): + if value is numpy_module: + glob[name] = _NumpyProxy(numpy_module) + clone = types.FunctionType(func.__code__, glob, func.__name__, func.__defaults__, func.__closure__) + clone.__kwdefaults__ = getattr(func, "__kwdefaults__", None) + return clone + + +# --- the tracing context ---------------------------------------------------- + + +class TraceContext: + def __init__(self, func): + self.func = func + self.nodes = [] + + def fail(self, msg, hint=None): + text = "{}{}".format(msg, _here(self.func)) + if hint: + text += "\n " + hint + raise _TraceError(text) + + def node(self, builder, children): + value = builder([c.value for c in children]) + return Sym(self, builder, children, value) + + def discard(self, sym): + # Identity, not equality: '==' on a Sym builds a comparison node. + for i, node in enumerate(self.nodes): + if node is sym: + del self.nodes[i] + return + + def leaf(self, value): + return Sym(self, lambda vals: value, [], value) + + def wrap(self, obj): + """Turn a Python value into a Sym, folding constants.""" + if isinstance(obj, Sym): + return obj + if isinstance(obj, emit.Value): + return self.leaf(obj) + value = emit.constant_value(obj, self.fail) + if value is None: + self.fail( + "Cannot use a value of type '{}' in a traced expression".format(type(obj).__name__), + "Only numbers, numpy scalars and one dimensional numpy arrays of numbers can be " + "combined with traced values.", + ) + return self.leaf(value) + + +def _check_no_control_flow(func, unroll): + """Refuse to trace a function whose source contains control flow.""" + if unroll: + return + try: + src = textwrap.dedent(inspect.getsource(func)) + tree = ast.parse(src) + except (OSError, TypeError, SyntaxError): + return # no source: rely on Sym.__bool__ to catch the data-dependent cases + for node in ast.walk(tree): + if isinstance(node, _CONTROL_FLOW_NODES): + raise _TraceError( + "The callable contains a '{}' statement, which the tracing backend cannot " + "translate.\n" + ' File "{}", line {}\n' + " Tracing records the operations that one Python run performs, so a branch is " + "either invisible (it depended on the arguments) or silently baked in (it did " + "not).\n" + " Use backend='ast' to translate the control flow, or pass unroll=True to " + "accept the Python-level evaluation.".format( + type(node).__name__.lower(), + getattr(func, "__code__", None) and func.__code__.co_filename, + node.lineno + (func.__code__.co_firstlineno - 1 if hasattr(func, "__code__") else 0), + ) + ) + + +# --- emission --------------------------------------------------------------- + + +def trace(func, param_names, param_types, declared_return, cpp_param_names, unroll=False): + """Trace the callable and return (body, deduced_return_type).""" + _check_no_control_flow(func, unroll) + + ctx = TraceContext(func) + args = [ctx.leaf(emit.Value(cpp_param_names[i], param_types[i])) for i in range(len(param_names))] + + try: + result = _with_traced_builtins(func)(*args) + except PyDeclareError: + raise + except TypeError as exc: + raise _TraceError( + "Tracing '{}' failed: {}\n This usually means an operation was applied to a traced " + "value that the tracing backend does not implement.".format(getattr(func, "__name__", ""), exc) + ) + + if isinstance(result, _MaybeMember): + result = result._node + if not isinstance(result, Sym): + value = emit.constant_value(result, ctx.fail) + if value is None: + ctx.fail( + "The traced callable returned a {} rather than a value derived from its arguments".format( + type(result).__name__ + ) + ) + result = ctx.leaf(value) + + lines, root_value = _emit(ctx, result) + + if declared_return is not None: + lines.append(" return {}::Return<{}>({});".format(emit.PYD, declared_return.cpp(), root_value.code)) + deduced = declared_return + else: + lines.append(" return {};".format(root_value.code)) + deduced = None if root_value.type.is_unknown else root_value.type + + return "\n".join(lines), deduced + + +def _emit(ctx, root): + """Emit the DAG, giving a name to every sub-expression used more than once.""" + # Count uses, reachable from the root only. + order = [] + seen = set() + + def visit(node): + if id(node) in seen: + node.uses += 1 + return + seen.add(id(node)) + node.uses = 1 + for child in node.children: + visit(child) + order.append(node) + + visit(root) + + lines = [] + values = {} + for node in order: + child_values = [values[id(c)] for c in node.children] + value = node.rebuild(child_values) + # A shared sub-expression is materialised so that it is computed once. + if node.uses > 1 and node.children: + name = "_pyd_t{}".format(len(lines) + 1) + lines.append(" const auto {} = {};".format(name, value.code)) + values[id(node)] = emit.Value(name, value.type, emit.PREC_PRIMARY) + else: + values[id(node)] = value + return lines, values[id(root)] diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py index ef7367548e3c2..21351044a3de3 100755 --- a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py @@ -11,7 +11,7 @@ import re import typing -from .._numbadeclare import _NumbaDeclareDecorator +from .._pydeclare import _numba_declare_dispatch class FunctionJitter: @@ -226,7 +226,7 @@ def jit_function(self, func, cols_list, extra_args): self.get_function_params_args_call(func, cols_list, extra_args) if not hasattr(self.func, "__cpp_wrapper__"): # not already declared with ROOT.Numba.Declare - _NumbaDeclareDecorator(self.func_sign, self.return_type)(self.func) + _numba_declare_dispatch(self.func_sign, self.return_type)(self.func) FunctionJitter.function_cache[self.func.__name__] = (self.func_call, self.func_sign) return self.func_call diff --git a/bindings/pyroot/pythonizations/test/CMakeLists.txt b/bindings/pyroot/pythonizations/test/CMakeLists.txt index e40a2175e84c4..e2bfbe22ea8ab 100644 --- a/bindings/pyroot/pythonizations/test/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/test/CMakeLists.txt @@ -117,6 +117,14 @@ endif() if (dataframe) # Test wrapping Python callables for use in C++ using numba ROOT_ADD_PYUNITTEST(pyroot_numbadeclare numbadeclare.py PYTHON_DEPS numba) + # The same suite, run against each implementation of Numba.Declare: the + # numba one and the two transpiler backends, which need no numba. + ROOT_ADD_PYUNITTEST(pyroot_pydeclare_numba pydeclare.py PYTHON_DEPS numba + ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=numba) + ROOT_ADD_PYUNITTEST(pyroot_pydeclare_ast pydeclare.py + ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=ast) + ROOT_ADD_PYUNITTEST(pyroot_pydeclare_trace pydeclare.py + ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=trace) ROOT_ADD_PYUNITTEST(pyroot_rdf_filter_pyz rdf_filter_pyz.py PYTHON_DEPS numba) ROOT_ADD_PYUNITTEST(pyroot_rdf_define_pyz rdf_define_pyz.py PYTHON_DEPS numba) # std::string_view in CPyCppyy diff --git a/bindings/pyroot/pythonizations/test/pydeclare.py b/bindings/pyroot/pythonizations/test/pydeclare.py new file mode 100644 index 0000000000000..d05cf25ab3435 --- /dev/null +++ b/bindings/pyroot/pythonizations/test/pydeclare.py @@ -0,0 +1,798 @@ +"""Backend-parameterised version of numbadeclare.py. + +The same suite is run against every implementation of ROOT.Numba.Declare that +exists: the original numba one, and the two transpiler backends that translate +the Python callable to C++ source instead. Which one is exercised is selected +by $ROOT_NUMBA_DECLARE_BACKEND, read by the decorator itself, so apart from the +handful of assertions that are about numba's implementation rather than about +the feature, this file is the original test suite unchanged. +""" + +import gc +import os +import platform +import sys +import unittest + +import numpy as np +import ROOT + +BACKEND = os.environ.get("ROOT_NUMBA_DECLARE_BACKEND", "numba").strip().lower() +IS_NUMBA = BACKEND in ("", "numba") + +if IS_NUMBA: + import numba as nb + +# The transpiler backends translate the callable to C++ source, so they are not +# restricted to the platforms the numba/cppyy bridge has been tested on. +numba_only = unittest.skipUnless(IS_NUMBA, "tests numba's implementation specifically") +needs_cppyy_numba_ext = unittest.skipIf( + IS_NUMBA and not (platform.system() == "Linux" and platform.machine() == "x86_64"), + "Numba integration is experimental and only tested on Linux x86_64", +) + +default_test_inputs = [-1.0, 0.0, 100.0] + + +class NumbaDeclareSimple(unittest.TestCase): + """ + Test decorator to create C++ wrapper for Python callables using numba with fundamental types + """ + + test_inputs = default_test_inputs + + # Test refcounts + def test_refcount_decorator(self): + """ + Test refcount of decorator + + In case of Python < 3.14, we expect a refcount of 2, because the call + to sys.getrefcount can create an additional reference itself. Starting + from Python 3.14, we expect a refcount of 1 because there were changes + to the interpreter to avoid some unnecessary ref counts. See also: + https://docs.python.org/3.14/whatsnew/3.14.html#whatsnew314-refcount + """ + x = ROOT.Numba.Declare(["float"], "float") + gc.collect() + extra_ref_count = int(sys.version_info < (3, 14)) + self.assertEqual(sys.getrefcount(x), 1 + extra_ref_count) + + @numba_only + def test_refcount_pycallable(self): + """ + Test refcount of decorated callable + """ + + def f1(x): + return x + + def f2(x): + return x + + fn0 = ROOT.Numba.Declare(["float"], "float")(f1) # noqa: F841 + ref = nb.cfunc("float32(float32)", nopython=True)(f2) # noqa: F841 + gc.collect() + self.assertEqual(sys.getrefcount(f1), sys.getrefcount(f2) + 2) + + # Test optional name + def test_optional_name(self): + """ + Test optional name of wrapper function + """ + optname = "optname2" + + @ROOT.Numba.Declare(["float"], "float", name=optname) + def f(x): + return x + + self.assertTrue(hasattr(ROOT.Numba, optname)) + + # Test attributes + def test_additional_attributes(self): + """ + Test additional attributes + """ + + @ROOT.Numba.Declare(["float"], "float") + def fn1(x): + return x + + gc.collect() + + # Every backend exposes the generated C++. + self.assertTrue(hasattr(fn1, "__cpp_wrapper__")) + self.assertTrue(isinstance(fn1.__cpp_wrapper__, str)) + # The generated code is held by the attribute, by the registry of + # declared names, and by cppjit, which keeps the last string argument + # of gInterpreter.Declare alive on the TInterpreter instance. + self.assertLessEqual(sys.getrefcount(fn1.__cpp_wrapper__), 4) + + if IS_NUMBA: + # The intermediate Python wrapper and the jitted callable only + # exist in the numba implementation. + self.assertTrue(hasattr(fn1, "__py_wrapper__")) + self.assertTrue(isinstance(fn1.__py_wrapper__, str)) + self.assertEqual(sys.getrefcount(fn1.__py_wrapper__), 2) + + self.assertTrue(hasattr(fn1, "numba_func")) + self.assertEqual(sys.getrefcount(fn1.numba_func), 3) + else: + self.assertEqual(fn1.__pydeclare_backend__, BACKEND) + self.assertEqual(fn1.__pydeclare_cpp_name__, "Numba::fn1") + + # Test cling integration + def test_cling(self): + """ + Test function call in cling + """ + + @ROOT.Numba.Declare(["float"], "float") + def fn12(x): + return 2.0 * x + + ROOT.gInterpreter.ProcessLine("y12 = Numba::fn12(42.0);") + self.assertEqual(fn12(42.0), ROOT.y12) + + # Test RDataFrame integration + def test_rdataframe(self): + """ + Test function call as part of RDataFrame + """ + + @ROOT.Numba.Declare(["unsigned int"], "float") + def fn13(x): + return 2.0 * x + + df = ROOT.RDataFrame(4).Define("x", "rdfentry_").Define("y", "Numba::fn13(x)") + mean_x = df.Mean("x") + mean_y = df.Mean("y") + self.assertEqual(mean_x.GetValue(), 1.5) + self.assertEqual(mean_y.GetValue(), 3.0) + + def test_rdataframe_temporary(self): + """ + Test passing a temporary from an RDataFrame operation + """ + + @ROOT.Numba.Declare(["const RVecF"], "RVecF") + def pass_temporary(v): + return v * np.array([2.0]).astype(np.float32) + + df = ROOT.RDataFrame(1).Define("v", "ROOT::RVecF{0.f,2.f}").Define("v2", "Numba::pass_temporary(v[v > 0])") + + rvecf = df.Take["RVecF"]("v2").GetValue()[0] + + self.assertTrue(np.array_equal(rvecf, np.array([4.0]))) + + def test_rdataframe_std_vector(self): + """ + Test function call as part of RDataFrame + """ + + @ROOT.Numba.Declare(["std::vector"], "std::vector") + def square_vec(x): + return x * x + + df = ROOT.RDataFrame(4).Define("x", "std::vector{1, 2, 3}").Define("x_sq", "Numba::square_vec(x)") + df.Display().Print() + self.assertEqual(df.Sum("x").GetValue(), 24) + self.assertEqual(df.Sum("x_sq").GetValue(), 56) + + def test_rdataframe_std_array(self): + """ + Test function call as part of RDataFrame with std::array + """ + + @ROOT.Numba.Declare(["std::array"], "std::array") + def square_array(x): + return x * x + + df = ROOT.RDataFrame(4).Define("x", "std::array{1, 2, 3}").Define("x_sq", "Numba::square_array(x)") + df.Display().Print() + self.assertEqual(df.Sum("x").GetValue(), 24) + self.assertEqual(df.Sum("x_sq").GetValue(), 56) + + @needs_cppyy_numba_ext + def test_rdataframe_LorentzVector(self): + """ + Test function call as part of RDataFrame with ROOT::Math::LorentzVector + """ + + @ROOT.Numba.Declare(["ROOT::Math::PtEtaPhiMVector"], "float") + def get_m(v): + return v.M() + + df = ROOT.RDataFrame(4).Define("v", "ROOT::Math::PtEtaPhiMVector(1, 2, 3, 4)").Define("v_m", "Numba::get_m(v)") + + self.assertEqual(df.Sum("v_m").GetValue(), 16.0) + + # Test wrappings + def test_wrapper_in_void(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare([], "float") + def fn2n(): + return float(42) + + x1 = fn2n() + x2 = ROOT.Numba.fn2n() + self.assertEqual(x1, x2) + self.assertEqual(type(x1), type(x2)) + + def test_wrapper_out_f(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["float"], "float") + def fn2(x): + return float(x) + + for v in self.test_inputs: + x1 = fn2(v) + x2 = ROOT.Numba.fn2(v) + self.assertEqual(x1, x2) + self.assertEqual(type(x1), type(x2)) + + def test_wrapper_out_d(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["float"], "double") + def fn2d(x): + return float(x) + + for v in self.test_inputs: + x1 = fn2d(v) + x2 = ROOT.Numba.fn2d(v) + self.assertEqual(x1, x2) + # NOTE: There is no double in Python because everything is a double. + self.assertEqual(type(x1), type(x2)) + + def test_wrapper_out_i(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["float"], "int") + def fn3(x): + return int(x) + + for v in self.test_inputs: + x1 = fn3(v) + x2 = ROOT.Numba.fn3(v) + self.assertEqual(x1, x2) + self.assertEqual(type(x1), type(x2)) + + def test_wrapper_out_l(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["float"], "long") + def fn4(x): + return x + + for v in self.test_inputs: + x1 = fn4(v) + x2 = ROOT.Numba.fn4(v) + self.assertEqual(x1, x2) + self.assertEqual(int, type(x2)) + + def test_wrapper_out_u(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["float"], "unsigned int") + def fn5(x): + return abs(x) + + for v in self.test_inputs: + x1 = fn5(v) + x2 = ROOT.Numba.fn5(v) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), int) + + def test_wrapper_out_k(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["float"], "unsigned long") + def fn6(x): + return abs(x) + + for v in self.test_inputs: + x1 = fn6(v) + x2 = ROOT.Numba.fn6(v) + self.assertEqual(x1, x2) + self.assertEqual(int, type(x2)) + + def test_wrapper_out_b(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["float"], "bool") + def fn6b(x): + return x > 0 + + for v in self.test_inputs: + x1 = fn6b(v) + x2 = ROOT.Numba.fn6b(v) + self.assertEqual(x1, x2) + self.assertEqual(type(x1), type(x2)) + + def test_wrapper_in_b(self): + """ + Test wrapper with different input/output configurations + + 'not x' is the one construct in this suite that the tracing backend + cannot translate: the interpreter resolves it through __bool__, which a + symbol cannot answer. The backend has to say so rather than guess, and + that is what is checked here. + """ + if BACKEND == "trace": + with self.assertRaises(Exception) as caught: + + @ROOT.Numba.Declare(["bool"], "bool") + def fn6b2_trace(x): + return not x + + self.assertIn("bool", str(caught.exception)) + return + + @ROOT.Numba.Declare(["bool"], "bool") + def fn6b2(x): + return not x + + for v in [True, False]: + x1 = fn6b2(v) + x2 = ROOT.Numba.fn6b2(v) + self.assertEqual(x1, x2) + self.assertEqual(type(x1), type(x2)) + + def test_wrapper_in_i(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["int"], "float") + def fn7i(x): + return 2.0 * x + + for v in [-1, 0, 1, 999]: + x1 = fn7i(v) + x2 = ROOT.Numba.fn7i(v) + self.assertEqual(x1, x2) + + def test_wrapper_in_l(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["long"], "float") + def fn7l(x): + return 2.0 * x + + for v in [-1, 0, 1, 999]: + x1 = fn7l(v) + x2 = ROOT.Numba.fn7l(v) + self.assertEqual(x1, x2) + + def test_wrapper_in_ui(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["unsigned int"], "float") + def fn7ui(x): + return 2.0 * x + + for v in [0, 1, 999]: + x1 = fn7ui(v) + x2 = ROOT.Numba.fn7ui(v) + self.assertEqual(x1, x2) + + def test_wrapper_in_ul(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["unsigned long"], "float") + def fn7ul(x): + return 2.0 * x + + for v in [0, 1, 999]: + x1 = fn7ul(v) + x2 = ROOT.Numba.fn7ul(v) + self.assertEqual(x1, x2) + + +class NumbaDeclareArray(unittest.TestCase): + """ + Test decorator to create C++ wrapper for Python callables using numba with RVecs + """ + + test_inputs = [default_test_inputs] + + # The global module index does not have RVec entities preloaded and + # gInterpreter.Declare is not allowed to load libROOTVecOps for RVec. + # Preload the library now. + ROOT.gSystem.Load("libROOTVecOps") + + def test_wrapper_in_vecf(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "float") + def g1(x): + return x.sum() + + for v in self.test_inputs: + x1 = g1(np.array(v, dtype=np.float32)) + x2 = ROOT.Numba.g1(ROOT.VecOps.RVec("float")(v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), float) + + def test_wrapper_in_vecf_vecd(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec", "RVec"], "float") + def g1_2vec(x, y): + return x.sum() + y.sum() + + for v in self.test_inputs: + x1 = g1_2vec(np.array(v, dtype=np.float32), np.array(v, dtype=np.float64)) + x2 = ROOT.Numba.g1_2vec(ROOT.VecOps.RVec("float")(v), ROOT.VecOps.RVec("double")(v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), float) + + def test_wrapper_in_vecd(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "float") + def g1d(x): + return x.sum() + + for v in self.test_inputs: + x1 = g1d(np.array(v, dtype=np.float64)) + x2 = ROOT.Numba.g1d(ROOT.VecOps.RVec("double")(v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), float) + + def test_wrapper_in_veci(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "int") + def g1i(x): + return x.sum() + + for v in self.test_inputs: + x1 = g1i(np.array(v, dtype=np.int32)) + x2 = ROOT.Numba.g1i(ROOT.VecOps.RVec("int")(int(x) for x in v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), int) + + def test_wrapper_in_vecl(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "int") + def g1l(x): + return x.sum() + + for v in self.test_inputs: + x1 = g1l(np.array(v, dtype=np.int64)) + x2 = ROOT.Numba.g1l(ROOT.VecOps.RVec("long")(int(x) for x in v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), int) + + def test_wrapper_in_vecui(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "int") + def g1ui(x): + return x.sum() + + for v in [[0, 1, 999]]: + x1 = g1ui(np.array(v, dtype=np.uint32)) + x2 = ROOT.Numba.g1ui(ROOT.VecOps.RVec("unsigned int")(v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), int) + + def test_wrapper_in_vecul(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "int") + def g1ul(x): + return x.sum() + + for v in [[0, 1, 999]]: + x1 = g1ul(np.array(v, dtype=np.uint64)) + x2 = ROOT.Numba.g1ul(ROOT.VecOps.RVec("unsigned long")(v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), int) + + def test_wrapper_in_vecb(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "int") + def g1b(x): + return x.sum() + + for v in [[True, False, True]]: + x1 = g1b(np.array(v, dtype=np.float32)) + x2 = ROOT.Numba.g1b(ROOT.VecOps.RVec("bool")(v)) + self.assertEqual(x1, x2) + self.assertEqual(type(x2), int) + + def test_wrapper_out_vecf(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "RVec") + def g2f(x): + return x[::-1] + + for v in [[0, 1, 999]]: + x1 = g2f(np.array(v, dtype=np.float32)) + x2 = ROOT.Numba.g2f(ROOT.VecOps.RVec("float")(v)) + self.assertTrue((x1 == x2).all()) + + def test_wrapper_out_vecd(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "RVec") + def g2d(x): + return x[::-1] + + for v in [[0, 1, 999]]: + x1 = g2d(np.array(v, dtype=np.float64)) + x2 = ROOT.Numba.g2d(ROOT.VecOps.RVec("double")(v)) + self.assertTrue((x1 == x2).all()) + + def test_wrapper_out_veci(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "RVec") + def g2i(x): + return x[::-1] + + for v in [[0, 1, 999]]: + x1 = g2i(np.array(v, dtype=np.int32)) + x2 = ROOT.Numba.g2i(ROOT.VecOps.RVec("int")(v)) + self.assertTrue((x1 == x2).all()) + + def test_wrapper_out_vecl(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "RVec") + def g2l(x): + return x[::-1] + + for v in [[0, 1, 999]]: + x1 = g2l(np.array(v, dtype=np.int64)) + x2 = ROOT.Numba.g2l(ROOT.VecOps.RVec("long")(v)) + self.assertTrue((x1 == x2).all()) + + def test_wrapper_out_vecul(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "RVec") + def g2ul(x): + return x[::-1] + + for v in [[0, 1, 999]]: + x1 = g2ul(np.array(v, dtype=np.uint64)) + x2 = ROOT.Numba.g2ul(ROOT.VecOps.RVec("unsigned long")(v)) + self.assertTrue((x1 == x2).all()) + + def test_wrapper_out_vecui(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "RVec") + def g2ui(x): + return x[::-1] + + for v in [[0, 1, 999]]: + x1 = g2ui(np.array(v, dtype=np.uint32)) + x2 = ROOT.Numba.g2ui(ROOT.VecOps.RVec("unsigned int")(v)) + self.assertTrue((x1 == x2).all()) + + def test_wrapper_out_vecb(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec"], "RVec") + def g2b(x): + return x[::-1] + + for v in [[True, False]]: + x1 = g2b(np.array(v, dtype=bool)) + x2 = ROOT.Numba.g2b(ROOT.VecOps.RVec("bool")(v)) + self.assertEqual(x1[0], bool(x2[0])) + self.assertEqual(x1[1], bool(x2[1])) + + def test_wrapper_in_vecfb_out_vecf(self): + """ + Test wrapper with different input/output configurations + """ + + @ROOT.Numba.Declare(["RVec", "RVec"], "RVec") + def g2fb(x, y): + return (x > 1) | y + + for vf, vb in [[[1.0, 2.0], [True, False]]]: + x1 = g2fb(np.array(vf, dtype=np.float32), np.array(vb, dtype=bool)) + x2 = ROOT.Numba.g2fb(ROOT.VecOps.RVec("float")(vf), ROOT.VecOps.RVec("bool")(vb)) + self.assertEqual(x1[0], bool(x2[0])) + self.assertEqual(x1[1], bool(x2[1])) + + def test_const_modifier(self): + """ + Test const modifier in input argument type + """ + + @ROOT.Numba.Declare(["const ROOT::VecOps::RVec"], "RVecF") + def const_mod(v): + return v * np.array([1.0, 2.0]).astype(np.float32) + + rvecf = ROOT.Numba.const_mod(ROOT.RVecF([1.0, 2.0])) + + self.assertTrue(np.array_equal(rvecf, np.array([1.0, 4.0]))) + + def test_reference(self): + """ + Test passing a reference as input argument + """ + + @ROOT.Numba.Declare(["RVec&"], "RVecF") + def pass_reference(v): + return v * np.array([1.0, 2.0]).astype(np.float32) + + rvecf = ROOT.Numba.pass_reference(ROOT.RVecF([1.0, 2.0])) + + self.assertTrue(np.array_equal(rvecf, np.array([1.0, 4.0]))) + + +class NumbaDeclareInferred(unittest.TestCase): + """ + Test decorator created with a reconstructed list of arguments using RDF column types, + and a return type inferred from the numba jitted function. + """ + + def test_fund_types(self): + """ + Test fundamental types + """ + df = ROOT.RDataFrame(4).Define("x", "rdfentry_") + + with self.subTest("function"): + + def is_even(x): + return x % 2 == 0 + + df = df.Define("is_even_x_1", is_even, ["x"]) + results = df.Take["bool"]("is_even_x_1").GetValue()[0] + self.assertEqual(results, True) + + with self.subTest("lambda"): + df = df.Define("is_even_x_2", lambda x: x % 2 == 0, ["x"]) + results = df.Take["bool"]("is_even_x_2").GetValue()[0] + self.assertEqual(results, True) + + def test_rvec(self): + """ + Test RVec + """ + df = ROOT.RDataFrame(4).Define("x", "ROOT::VecOps::RVec({1, 2, 3})") + + with self.subTest("function"): + + def square_rvec(v): + return v * v + + df = df.Define("square_rvec_1", square_rvec, ["x"]) + results = df.Take["RVec"]("square_rvec_1").GetValue()[0] + self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) + + with self.subTest("lambda"): + df = df.Define("square_rvec_2", lambda v: v * v, ["x"]) + results = df.Take["RVec"]("square_rvec_2").GetValue()[0] + self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) + + def test_std_vec(self): + """ + Test std::vector + """ + df = ROOT.RDataFrame(4).Define("x", "std::vector({1, 2, 3})") + + with self.subTest("function"): + + def square_std_vec(v): + return v * v + + df = df.Define("square_std_vec_1", square_std_vec, ["x"]) + results = df.Take["RVec"]("square_std_vec_1").GetValue()[0] + self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) + + with self.subTest("lambda"): + df = df.Define("square_std_vec_2", lambda v: v * v, ["x"]) + results = df.Take["RVec"]("square_std_vec_2").GetValue()[0] + self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) + + def test_std_array(self): + """ + Test std::array + """ + df = ROOT.RDataFrame(4).Define("x", "std::array({1, 2, 3})") + + with self.subTest("function"): + + def square_std_arr(v): + return v * v + + df = df.Define("square_std_arr_1", square_std_arr, ["x"]) + results = df.Take["RVec"]("square_std_arr_1").GetValue()[0] + self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) + + with self.subTest("lambda"): + df = df.Define("square_std_arr_2", lambda v: v * v, ["x"]) + results = df.Take["RVec"]("square_std_arr_2").GetValue()[0] + self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) + + def test_missing_signature(self): + """ + A method call on a C++ object, with no return type given. + + numba cannot infer the return type here and raises. The transpiler + backends do not have to infer it: they emit the call and let the C++ + compiler deduce the type, so the same code just works. + """ + + def f(x): + return x.M() + + df = ROOT.RDataFrame(4).Define("v", "ROOT::Math::PtEtaPhiMVector(1, 2, 3, 4)") + + if IS_NUMBA: + with self.assertRaises(Exception): + df.Define("m", f, ["v"]) + else: + masses = df.Define("m", f, ["v"]).Take["double"]("m").GetValue() + self.assertAlmostEqual(masses[0], 4.0) + + +if __name__ == "__main__": + unittest.main() From 1114ad21f1d6acac57556c8d6c301d404c513019 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 30 Aug 2026 21:13:56 +0000 Subject: [PATCH 2/8] [Python] Let translated code name C++ entities through cppyy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ROOT.TMath.Abs(x) and friends now translate to the C++ call, in both backends: the AST one walks the namespace with getattr so an unknown name is reported at declaration time, and the tracer sees a proxy that records the call when any argument is symbolic and does the real thing otherwise. Global C++ objects such as gRandom are refused rather than turned into a class-qualified call, which would compile but mean something else. 🤖 Done with the help of AI --- .../python/ROOT/_pydeclare/README.md | 5 +- .../python/ROOT/_pydeclare/ast_backend.py | 55 +++++++++++++------ .../python/ROOT/_pydeclare/emit.py | 39 +++++++++++++ .../python/ROOT/_pydeclare/trace_backend.py | 53 ++++++++++++++++-- 4 files changed, 128 insertions(+), 24 deletions(-) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md index 75cb0573f5618..86e51813a06ab 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md @@ -47,8 +47,9 @@ resolves them. **Expressions.** Arithmetic, comparisons (including chained ones on scalars), `and`/`or`/`not`, the bitwise operators, indexing with wrap-around for negative indices, slicing with full Python semantics, boolean-mask indexing (`v[v > 0]`), -conditional expressions, calls to other declared callables and to cppyy -entities such as `ROOT.TMath.Abs`. +conditional expressions, calls to other declared callables, and calls to +cppyy entities such as `ROOT.TMath.Abs` -- the C++ namespace is walked through +cppyy, so a name cling does not know is reported at declaration time. **Statements** (`ast` backend only): assignment, augmented and annotated assignment, `if`/`elif`/`else`, `for` over a range or an array, `while`, diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py index 414323779dbfe..c8b27997f9ae0 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py @@ -68,8 +68,9 @@ def __init__(self, name, path): class _CppRef: """A cppyy entity (class, namespace or free function) usable from C++.""" - def __init__(self, cpp_name): + def __init__(self, cpp_name, obj=None): self.cpp_name = cpp_name + self.obj = obj class _DeclaredRef: @@ -613,7 +614,7 @@ def ex_Attribute(self, node): if isinstance(base, _ModuleRef): return self._module_attribute(base, name, node) if isinstance(base, _CppRef): - return _CppRef("{}::{}".format(base.cpp_name, name)) + return self._cpp_attribute(base, name, node) if isinstance(base, _DeclaredRef): raise_unsupported(self.func, node, "Cannot take an attribute of a declared function") @@ -636,6 +637,33 @@ def ex_Attribute(self, node): # Opaque C++ object: pass the access straight through. return _CppAttr(value, name) + def _cpp_attribute(self, base, name, node): + """Walk one step further into the C++ namespace, validating as we go.""" + child = None + if base.obj is not None: + try: + child = getattr(base.obj, name) + except AttributeError: + raise_unsupported( + self.func, + node, + "'{}' has no member '{}' known to cling".format(base.cpp_name or "the global namespace", name), + ) + if emit.cpp_entity_kind(child) is None: + raise_unsupported( + self.func, + node, + "'{}{}' is a C++ object, not a type or a function".format( + base.cpp_name + "::" if base.cpp_name else "", name + ), + hint="Only classes, namespaces and functions can be named from translated " + "code. Pass the object in as an argument instead.", + ) + qualified = emit.cpp_entity_name(child) if child is not None else None + if qualified is None: + qualified = "{}::{}".format(base.cpp_name, name) if base.cpp_name else name + return _CppRef(qualified, child) + def ex_Call(self, node): fail = self.fail_at(node) emit.check_no_kwargs(node.keywords, "a translated function", fail) @@ -859,6 +887,9 @@ def _classify(self, obj, path, node): if value is not None: return value + if emit.is_cpp_root_namespace(obj): + return _CppRef("", obj) + module_name = getattr(obj, "__name__", None) if inspect.ismodule(obj): if module_name in ("numpy", "math"): @@ -874,9 +905,10 @@ def _classify(self, obj, path, node): if declared is not None: return _DeclaredRef(declared, getattr(obj, "__pydeclare_return_type__", None)) - cpp_name = _cppyy_name(obj) - if cpp_name is not None: - return _CppRef(cpp_name) + if emit.cpp_entity_kind(obj) is not None: + cpp_name = emit.cpp_entity_name(obj) + if cpp_name is not None: + return _CppRef(cpp_name, obj) if callable(obj) and module_name in emit.UNARY_FUNCTIONS: return _FuncRef(module_name, path) @@ -1041,16 +1073,3 @@ def _unify(a, b): return UNKNOWN_T merged = ct.promote(a, b) return merged - - -def _cppyy_name(obj): - """The C++ name of a cppyy class, namespace or function, if it is one.""" - name = getattr(obj, "__cpp_name__", None) - if isinstance(name, str) and name: - return name - module = getattr(obj, "__module__", "") or "" - qual = getattr(obj, "__name__", None) - if isinstance(qual, str) and qual and (module.startswith("cppyy") or module == "ROOT"): - prefix = module.split(".", 1)[1] if module.startswith("cppyy.gbl.") else "" - return "{}::{}".format(prefix, qual) if prefix else qual - return None diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py index aaafbd6655cd0..94e61d26ffd30 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py @@ -629,6 +629,45 @@ def astype(obj, dtype_obj, fail): ) +# --------------------------------------------------------------------------- +# cppyy entities +# +# Anything cling already knows can be named directly from translated code; the +# transpiler needs no model of it beyond a valid C++ name. +# --------------------------------------------------------------------------- + + +def is_cpp_root_namespace(obj): + """True for the ROOT facade and cppyy.gbl, the roots of the C++ world.""" + if type(obj).__name__ == "ROOTFacade": + return True + return getattr(obj, "__name__", None) == "gbl" and (getattr(obj, "__module__", "") or "").startswith("cppyy") + + +def cpp_entity_name(obj): + """The C++ name of a cppyy class or namespace, if it has a usable one. + + Function overloads report a signature rather than a name, so they fall + back to being addressed by the path they were reached through. + """ + name = getattr(obj, "__cpp_name__", None) + if isinstance(name, str) and name and "(" not in name: + return name + return None + + +def cpp_entity_kind(obj): + """'type' for a class or namespace, 'callable' for a function, else None.""" + if isinstance(obj, type): + return "type" + kind = type(obj).__name__ + if kind in ("CPPOverload", "TemplateProxy", "CPPFunction", "StaticCPPOverload"): + return "callable" + if callable(obj) and (type(obj).__module__ or "").startswith("cppyy"): + return "callable" + return None + + def check_no_kwargs(kwargs, what, fail): if kwargs: fail("Keyword arguments are not supported in a call to {}".format(what)) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py index 50b93398de511..b38b47a67b21e 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py @@ -29,6 +29,7 @@ import textwrap from . import emit +from .cpptypes import UNKNOWN_T from .errors import PyDeclareError _CONTROL_FLOW_NODES = (ast.If, ast.For, ast.While, ast.Try, ast.With, ast.IfExp) @@ -479,6 +480,49 @@ def build(vals): return ctx.node(build, children) +class _CppProxy: + """Stands in for a C++ namespace, class or function while tracing. + + Calling it with a symbol records a C++ call; calling it with ordinary + Python values does the real thing, so a traced callable that happens to use + cppyy for something unrelated keeps working. + """ + + def __init__(self, cpp_name, obj): + object.__setattr__(self, "_cpp_name", cpp_name) + object.__setattr__(self, "_obj", obj) + + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + obj = object.__getattribute__(self, "_obj") + prefix = object.__getattribute__(self, "_cpp_name") + try: + child = getattr(obj, name) + except AttributeError: + raise _TraceError("'{}' has no member '{}' known to cling".format(prefix or "the global namespace", name)) + if emit.cpp_entity_kind(child) is None: + raise _TraceError( + "'{}{}' is a C++ object, not a type or a function.\n" + " Only classes, namespaces and functions can be named from translated code. " + "Pass the object in as an argument instead.".format(prefix + "::" if prefix else "", name) + ) + qualified = emit.cpp_entity_name(child) or ("{}::{}".format(prefix, name) if prefix else name) + return _CppProxy(qualified, child) + + def __call__(self, *args, **kwargs): + obj = object.__getattribute__(self, "_obj") + syms = [a for a in args if isinstance(a, Sym)] + if not syms: + return obj(*args, **kwargs) + if kwargs: + raise _TraceError("Keyword arguments are not supported in a call to C++") + ctx = syms[0].ctx + name = object.__getattribute__(self, "_cpp_name") + children = [ctx.wrap(a) for a in args] + return ctx.node(lambda vals: emit.call(name, vals, UNKNOWN_T), children) + + TRACED_BUILTINS = ("float", "int", "bool", "len", "abs", "round", "sum", "pow", "min", "max") @@ -495,10 +539,11 @@ def _with_traced_builtins(func): glob = dict(getattr(func, "__globals__", {}) or {}) glob["__builtins__"] = env numpy_module = sys.modules.get("numpy") - if numpy_module is not None: - for name, value in list(glob.items()): - if value is numpy_module: - glob[name] = _NumpyProxy(numpy_module) + for name, value in list(glob.items()): + if numpy_module is not None and value is numpy_module: + glob[name] = _NumpyProxy(numpy_module) + elif emit.is_cpp_root_namespace(value): + glob[name] = _CppProxy("", value) clone = types.FunctionType(func.__code__, glob, func.__name__, func.__defaults__, func.__closure__) clone.__kwdefaults__ = getattr(func, "__kwdefaults__", None) return clone From 314151c33c5fa059aff1c55d2cd9ab83963dbcd3 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 30 Aug 2026 21:47:23 +0000 Subject: [PATCH 3/8] [Python] Use the VecOps reductions where numpy and ROOT agree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyD::Sum and friends were hand-written loops, which cost about 10% against the equivalent ROOT::VecOps call in the RDataFrame event loop. Only the boolean case actually needs different semantics -- numpy sums booleans as integers where VecOps::Sum of an RVec is a bool -- so everything else now forwards to VecOps and the generated code performs exactly like the hand-written C++ it replaces. Also drop the tracer's PyD::Return wrapper when the type already matches, as the AST backend does, so both backends emit identical code. 🤖 Done with the help of AI --- .../python/ROOT/_pydeclare/support.py | 320 ++++++++++++++---- .../python/ROOT/_pydeclare/trace_backend.py | 5 +- 2 files changed, 259 insertions(+), 66 deletions(-) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py index 1e2a9a3aa2775..6dbd271b528e8 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/support.py @@ -31,9 +31,11 @@ #include #include #include +#include #include #include #include +#include #include namespace ROOT { @@ -86,6 +88,96 @@ } } +/// The type an integer operation is carried out in. +/// +/// Python integers do not overflow and have no unsigned flavour, so the usual +/// C++ arithmetic conversions are wrong for us in one specific place: a mix of +/// a signed and an unsigned operand converts the signed one, turning -7 into a +/// huge positive value before the operation ever runs. Working in a signed +/// 64-bit type whenever either operand is signed reproduces Python for every +/// pair of operands that is narrower than 64 bits, which is all of them except +/// long long itself. +template +using IntOpType = + typename std::conditional::value || std::is_signed::value, + typename std::conditional<(sizeof(A) < sizeof(long long) && + sizeof(B) < sizeof(long long)), + long long, typename std::common_type::type>::type, + typename std::common_type::type>::type; + +/// The type a Python arithmetic expression on A and B produces. +template +using ArithResult = + typename std::conditional::value && std::is_integral::value, IntOpType, + typename std::common_type::type>::type; + +/// '<' that is correct across a signed/unsigned mix, where C++'s own '<' is not. +template +constexpr bool Less(A a, B b) +{ + if constexpr (std::is_integral::value && std::is_integral::value && + std::is_signed::value != std::is_signed::value) { + if constexpr (std::is_signed::value) { + if (a < A(0)) { + return true; + } + return static_cast::type>(a) < b; + } else { + if (b < B(0)) { + return false; + } + return a < static_cast::type>(b); + } + } else { + return a < b; + } +} + +/// Python's unary minus. Negating an unsigned value in C++ wraps around; +/// Python has no unsigned integers and simply produces a negative number. +template +auto NegScalar(T x) +{ + if constexpr (std::is_integral::value && !std::is_signed::value) { + return -static_cast>(x); + } else { + return -x; + } +} + +template +auto Neg(const T &x) +{ + if constexpr (IsVecLike::type>::value) { + using E = ElemType; + using R = typename std::decay()))>::type; + ROOT::VecOps::RVec out(x.size()); + for (std::size_t i = 0; i < x.size(); ++i) { + out[i] = NegScalar(static_cast(x[i])); + } + return out; + } else { + return NegScalar(x); + } +} + +/// Python's 'and' and 'or' evaluate to one of their *operands*, not to a bool, +/// and they short-circuit. The right-hand side is passed as a callable so +/// that it is only evaluated when Python would evaluate it. +template +auto And(const A &a, F &&rhs) +{ + using R = typename std::common_type::type; + return a ? static_cast(rhs()) : static_cast(a); +} + +template +auto Or(const A &a, F &&rhs) +{ + using R = typename std::common_type::type; + return a ? static_cast(a) : static_cast(rhs()); +} + // --------------------------------------------------------------------------- // Adapting the supported container types to RVec // @@ -119,6 +211,14 @@ return ROOT::VecOps::RVec(const_cast(v.data()), N); } +// The views above are non-owning, so a view over a temporary would dangle as +// soon as the full expression ends. Generated code never does this, but the +// header is declared into a live interpreter where anyone can call AsRVec. +template +ROOT::VecOps::RVec AsRVec(std::vector &&) = delete; +template +ROOT::VecOps::RVec AsRVec(std::array &&) = delete; + // --------------------------------------------------------------------------- // Element-wise application over a mix of containers and scalars // --------------------------------------------------------------------------- @@ -198,18 +298,44 @@ } else if constexpr (IsVecLike::type>::value) { return static_cast(a) / AsDoubleVec(b); } else { + if constexpr (std::is_integral::value && std::is_integral::value) { + if (b == B(0)) { + throw std::domain_error("PyDeclare: division by zero"); + } + } return static_cast(a) / static_cast(b); } } +/// Guard the two integer divisions C++ leaves undefined. Both of them abort +/// the process on x86 rather than producing a wrong number, so there is no +/// choice about paying for the check; Python raises here in any case. +template +void CheckIntDivisor(T a, T b) +{ + if (b == T(0)) { + throw std::domain_error("PyDeclare: integer division or modulo by zero"); + } + if constexpr (std::is_signed::value) { + if (b == T(-1) && a == std::numeric_limits::min()) { + throw std::overflow_error("PyDeclare: integer division overflow; the result of " + "-(-2**63) is not representable as a C++ integer"); + } + } +} + /// Python's '//' floors; C++ integer division truncates towards zero. template auto FloorDivScalar(A a, B b) { if constexpr (std::is_integral::value && std::is_integral::value) { - auto q = a / b; - auto r = a % b; - if (r != 0 && (IsNeg(r) != IsNeg(b))) { + using T = IntOpType; + const T x = static_cast(a); + const T y = static_cast(b); + CheckIntDivisor(x, y); + T q = x / y; + const T r = x % y; + if (r != 0 && (IsNeg(r) != IsNeg(y))) { q -= 1; } return q; @@ -235,9 +361,13 @@ auto ModScalar(A a, B b) { if constexpr (std::is_integral::value && std::is_integral::value) { - auto r = a % b; - if (r != 0 && (IsNeg(r) != IsNeg(b))) { - r += b; + using T = IntOpType; + const T x = static_cast(a); + const T y = static_cast(b); + CheckIntDivisor(x, y); + T r = x % y; + if (r != 0 && (IsNeg(r) != IsNeg(y))) { + r += y; } return r; } else { @@ -270,8 +400,11 @@ throw std::runtime_error("PyDeclare: integer ** negative integer would return a float in Python; " "cast a base or exponent to a floating point type"); } - A result = A(1); - A base = a; + // Accumulating in A would truncate: a short base overflows at 2**15 + // even though both Python and C++'s own integer promotion would not. + using T = IntOpType; + T result = T(1); + T base = static_cast(a); auto e = b; while (e > 0) { if (e & 1) { @@ -303,36 +436,35 @@ // Indexing and slicing with Python semantics // --------------------------------------------------------------------------- -template -decltype(auto) Index(const V &v, I i) +/// Resolve a Python index against a size, wrapping negatives and rejecting +/// anything out of range. Python raises IndexError rather than reading past +/// the end, and an out-of-bounds read is not something to hand to an analysis. +template +std::size_t IndexOf(I i, std::size_t size) { + const long long n = static_cast(size); + long long j = static_cast(i); if constexpr (std::is_signed::value) { - if (i < 0) { - const long long n = static_cast(v.size()); - const long long j = n + static_cast(i); - if (j < 0) { - throw std::out_of_range("PyDeclare: index out of range"); - } - return v[static_cast(j)]; + if (j < 0) { + j += n; } } - return v[static_cast(i)]; + if (j < 0 || j >= n) { + throw std::out_of_range("PyDeclare: index out of range"); + } + return static_cast(j); +} + +template +decltype(auto) Index(const V &v, I i) +{ + return v[IndexOf(i, v.size())]; } template decltype(auto) Index(V &v, I i) { - if constexpr (std::is_signed::value) { - if (i < 0) { - const long long n = static_cast(v.size()); - const long long j = n + static_cast(i); - if (j < 0) { - throw std::out_of_range("PyDeclare: index out of range"); - } - return v[static_cast(j)]; - } - } - return v[static_cast(i)]; + return v[IndexOf(i, v.size())]; } /// Full Python slice semantics, including negative bounds and negative steps. @@ -382,28 +514,51 @@ // Reductions, with numpy's result conventions // --------------------------------------------------------------------------- +/// The type numpy accumulates a sum or a product into: at least 64 bits for +/// any integer input, so that summing an array of small integers does not +/// overflow the element type. ROOT::VecOps::Sum accumulates into the element +/// type instead, which is why the integer cases below are spelled out. +template +using AccType = + typename std::conditional::value, + typename std::conditional::value || + std::is_same::value, + long long, unsigned long long>::type, + E>::type; + +/// numpy sums booleans and narrow integers as wide integers. For floating +/// point the two agree, and the VecOps implementation is used unchanged so +/// that the generated code is exactly as fast as hand-written C++. template auto Sum(const V &v) { using E = ElemType; - using R = typename std::conditional::value, long, E>::type; - R acc = R(0); - for (std::size_t i = 0; i < v.size(); ++i) { - acc += static_cast(v[i]); + if constexpr (std::is_integral::value) { + using Acc = AccType; + Acc acc = 0; + for (std::size_t i = 0; i < v.size(); ++i) { + acc += static_cast(v[i]); + } + return acc; + } else { + return ROOT::VecOps::Sum(v); } - return acc; } template auto Prod(const V &v) { using E = ElemType; - using R = typename std::conditional::value, long, E>::type; - R acc = R(1); - for (std::size_t i = 0; i < v.size(); ++i) { - acc *= static_cast(v[i]); + if constexpr (std::is_integral::value) { + using Acc = AccType; + Acc acc = 1; + for (std::size_t i = 0; i < v.size(); ++i) { + acc *= static_cast(v[i]); + } + return acc; + } else { + return ROOT::VecOps::Product(v); } - return acc; } template @@ -412,11 +567,7 @@ if (v.empty()) { return std::numeric_limits::quiet_NaN(); } - double acc = 0.; - for (std::size_t i = 0; i < v.size(); ++i) { - acc += static_cast(v[i]); - } - return acc / static_cast(v.size()); + return ROOT::VecOps::Mean(v); } /// numpy's std(), i.e. the population standard deviation (ddof = 0). Note @@ -439,33 +590,19 @@ template auto Min(const V &v) { - using E = ElemType; if (v.empty()) { throw std::runtime_error("PyDeclare: min() of an empty sequence"); } - E best = v[0]; - for (std::size_t i = 1; i < v.size(); ++i) { - if (v[i] < best) { - best = v[i]; - } - } - return best; + return ROOT::VecOps::Min(v); } template auto Max(const V &v) { - using E = ElemType; if (v.empty()) { throw std::runtime_error("PyDeclare: max() of an empty sequence"); } - E best = v[0]; - for (std::size_t i = 1; i < v.size(); ++i) { - if (best < v[i]) { - best = v[i]; - } - } - return best; + return ROOT::VecOps::Max(v); } template @@ -534,7 +671,6 @@ return FN(x); \ } -PYDECLARE_UNARY_MATH(Abs, abs) PYDECLARE_UNARY_MATH(Sqrt, sqrt) PYDECLARE_UNARY_MATH(Cbrt, cbrt) PYDECLARE_UNARY_MATH(Exp, exp) @@ -566,6 +702,36 @@ #undef PYDECLARE_UNARY_MATH +/// abs() is the one function here with no unsigned overload in : for +/// an unsigned argument every candidate is an equally good conversion and the +/// call is ambiguous. Python's abs() of a non-negative number is the identity. +template +auto AbsScalar(T x) +{ + if constexpr (std::is_integral::value && !std::is_signed::value) { + return x; + } else { + using std::abs; + return abs(x); + } +} + +template +auto Abs(const T &x) +{ + if constexpr (IsVecLike::type>::value) { + using E = ElemType; + if constexpr (std::is_integral::value && !std::is_signed::value) { + return x; + } else { + using namespace ROOT::VecOps; + return abs(x); + } + } else { + return AbsScalar(x); + } +} + /// Python's round() and numpy's round() both round halves to even, while /// std::round rounds halves away from zero. std::nearbyint under the default /// rounding mode does what Python does. @@ -604,14 +770,28 @@ #undef PYDECLARE_BINARY_MATH +template +auto MaximumScalar(A a, B b) +{ + using R = ArithResult; + return Less(a, b) ? static_cast(b) : static_cast(a); +} + +template +auto MinimumScalar(A a, B b) +{ + using R = ArithResult; + return Less(b, a) ? static_cast(b) : static_cast(a); +} + template auto Maximum(const A &a, const B &b) { if constexpr (IsVecLike::type>::value || IsVecLike::type>::value) { - return ZipMap(a, b, [](auto x, auto y) { return x > y ? x : y; }); + return ZipMap(a, b, [](auto x, auto y) { return MaximumScalar(x, y); }); } else { - return a > b ? a : b; + return MaximumScalar(a, b); } } @@ -620,9 +800,9 @@ { if constexpr (IsVecLike::type>::value || IsVecLike::type>::value) { - return ZipMap(a, b, [](auto x, auto y) { return x < y ? x : y; }); + return ZipMap(a, b, [](auto x, auto y) { return MinimumScalar(x, y); }); } else { - return a < b ? a : b; + return MinimumScalar(a, b); } } @@ -631,6 +811,16 @@ { if constexpr (IsVecLike::type>::value) { const std::size_t n = c.size(); + if constexpr (IsVecLike::type>::value) { + if (a.size() != n) { + throw std::runtime_error("PyDeclare: operands could not be broadcast together"); + } + } + if constexpr (IsVecLike::type>::value) { + if (b.size() != n) { + throw std::runtime_error("PyDeclare: operands could not be broadcast together"); + } + } using R = typename std::decay::type; ROOT::VecOps::RVec out(n); for (std::size_t i = 0; i < n; ++i) { diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py index b38b47a67b21e..5d68aab26d329 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py @@ -655,7 +655,10 @@ def trace(func, param_names, param_types, declared_return, cpp_param_names, unro lines, root_value = _emit(ctx, result) if declared_return is not None: - lines.append(" return {}::Return<{}>({});".format(emit.PYD, declared_return.cpp(), root_value.code)) + if root_value.type == declared_return: + lines.append(" return {};".format(root_value.code)) + else: + lines.append(" return {}::Return<{}>({});".format(emit.PYD, declared_return.cpp(), root_value.code)) deduced = declared_return else: lines.append(" return {};".format(root_value.code)) From 2c98d9f8a44127d98c3635193eeff0ebf866f355 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 31 Aug 2026 05:13:37 +0000 Subject: [PATCH 4/8] [Python] Drop the tracing backend of the Python-to-C++ transpiler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AST backend turned out strictly more capable. It translates control flow, which the tracer refuses because it cannot see it, and it reports an unsupported construct by name rather than only when the recorded expression fails to compile. The tracer's one advantage -- not needing the Python source -- does not pay for a second implementation of the same feature. The lowering layer stays split from the tree walk: emit.py decides what each Python operation becomes in C++, ast_backend.py only discovers which operations there are. The tracer remains in the history if it is ever wanted back. 🤖 Done with the help of AI --- .../pythonizations/python/CMakeLists.txt | 1 - .../python/ROOT/_pydeclare/README.md | 89 ++- .../python/ROOT/_pydeclare/__init__.py | 71 +- .../python/ROOT/_pydeclare/ast_backend.py | 189 ++++- .../python/ROOT/_pydeclare/emit.py | 127 +++- .../python/ROOT/_pydeclare/errors.py | 4 +- .../python/ROOT/_pydeclare/trace_backend.py | 700 ------------------ .../pyroot/pythonizations/test/CMakeLists.txt | 6 +- .../pyroot/pythonizations/test/pydeclare.py | 23 +- 9 files changed, 356 insertions(+), 854 deletions(-) delete mode 100644 bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py diff --git a/bindings/pyroot/pythonizations/python/CMakeLists.txt b/bindings/pyroot/pythonizations/python/CMakeLists.txt index 33496b9644516..bfba62dcb74ef 100644 --- a/bindings/pyroot/pythonizations/python/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/python/CMakeLists.txt @@ -89,7 +89,6 @@ set(py_sources ROOT/_pydeclare/emit.py ROOT/_pydeclare/errors.py ROOT/_pydeclare/support.py - ROOT/_pydeclare/trace_backend.py ROOT/_pythonization/__init__.py ROOT/_pythonization/_cppinstance.py ROOT/_pythonization/_drawables.py diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md index 86e51813a06ab..5a47df877dc98 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md @@ -14,27 +14,19 @@ ROOT.RDataFrame(4).Define("x", "(float)rdfentry_").Define("y", "Py::pypow(x, 3)" The generated code is on the callable as `__cpp_wrapper__`. -## The two backends +## How it works -Both lower through the same module (`emit.py`), so for input they both accept -they emit identical C++. +`ast_backend.py` walks the Python syntax tree and discovers which operations +the callable performs. `emit.py` decides what each one becomes in C++, and +`support.py` holds the C++ templates the generated code calls. Splitting +discovery from lowering means every semantic decision is written down once. -| | `ast` (default) | `trace` | -|---|---|---| -| How | walks the Python syntax tree | runs the callable once on symbolic arguments | -| Needs the source | yes | no | -| `if` / `for` / `while` / early return | translated | refused | -| `not x`, `x and y`, `x or y` | translated | refused (resolved via `__bool__`) | -| Operations it has never heard of | refused by name | often works, C++ decides | - -Select with `backend=`, or `$ROOT_PYDECLARE_BACKEND`. The tracer refuses -control flow because it cannot see it: a branch is either invisible (it -depended on an argument) or silently baked in (it did not). Pass `unroll=True` -to accept the second case deliberately. +The callable's argument types are known up front -- RDataFrame knows its column +types, and the decorator takes them explicitly -- so forward type inference +through the tree is enough to choose the right lowering everywhere. -`ROOT.Numba.Declare` can be routed to either backend with -`$ROOT_NUMBA_DECLARE_BACKEND=ast|trace`, which keeps existing `"Numba::f(x)"` -call strings working. +`ROOT.Numba.Declare` is the same translation declared into the `Numba` C++ +namespace, so existing `"Numba::f(x)"` call strings keep working. ## The supported subset @@ -51,8 +43,7 @@ conditional expressions, calls to other declared callables, and calls to cppyy entities such as `ROOT.TMath.Abs` -- the C++ namespace is walked through cppyy, so a name cling does not know is reported at declaration time. -**Statements** (`ast` backend only): assignment, augmented and annotated -assignment, `if`/`elif`/`else`, `for` over a range or an array, `while`, +**Statements.** Assignment, augmented and annotated assignment, `if`/`elif`/`else`, `for` over a range or an array, `while`, `break`, `continue`, `pass`, `assert`, early `return`. **Builtins.** `abs`, `bool`, `float`, `int`, `len`, `max`, `min`, `pow`, @@ -65,8 +56,10 @@ assignment, `if`/`elif`/`else`, `for` over a range or an array, `while`, **Constants.** Any expression built only from module-level or closure variables is evaluated once at declaration time and inlined, as numba freezes globals. -Only numpy, math and the pure builtins are allowed to run during that -evaluation, so a call like `os.getpid()` is refused rather than frozen. +Only *pure* numpy and math functions and the pure builtins are allowed to run +during that evaluation: `os.getpid()` is refused rather than frozen, and so are +`np.random` and the numpy functions that read files, which would otherwise be +folded into a literal that every entry then shares. ## Where Python and C++ disagree @@ -80,20 +73,50 @@ templates in `support.py`: | `-7 % 2` is `1` | `-1` | `PyD::Mod` (skipped for unsigned, where they agree) | | `2 ** 10` is an `int` | `std::pow` returns a double | `PyD::Pow` | | `v[-1]` is the last element | out of bounds | `PyD::Index` | +| `v[99]` of a shorter array raises | out of bounds | `PyD::Index` (checked) | | `round(0.5)` is `0` | `std::round` gives `1` | `PyD::Round` (half to even) | -| `sum` of a bool array is an `int` | `Sum` of `RVec` is a `bool` | `PyD::Sum` | +| `sum` of an `int8` array is a 64-bit int | `Sum` accumulates in `int8` | `PyD::Sum` | +| `-1 < 2` with an unsigned `2` | converts `-1` to a huge value | widened to `long long` | +| `-x` of an unsigned `x` is negative | wraps around | `PyD::Neg` | +| `7 // 0` raises | undefined, aborts the process | `PyD::FloorDiv` (throws) | +| `0.0 or 5.0` is `5.0` | `||` gives `true` | `PyD::Or` / `PyD::And` | +| `np.where` needs equal lengths | reads past the end | `PyD::Where` (checked) | + +Two disagreements are left in place deliberately, because the C++ behaviour is +the one a physicist wants and checking on every element would cost more than it +is worth: + +* Floating point division by zero gives `inf`/`nan` rather than raising, as IEEE + 754 prescribes. Integer division by zero does raise, because the alternative + is not a wrong number but a `SIGFPE` that takes the process down. +* `min`, `max`, `argmin` and `argmax` skip NaN, following `ROOT::VecOps`, where + numpy propagates it. `mean` and `std` do propagate it. ## Deliberate restrictions -Local variables are declared `auto`, so the type is always the one the C++ -compiler deduces, never one this package guessed. A variable may not change -type between assignments, and one assigned inside a nested block is hoisted to -function scope -- which is Python's own scoping -- and therefore needs a type -that can be determined up front; annotate it (`total: 'double' = 0.0`) if it -cannot be. +Local variables are declared `auto`, so the type is the one the C++ compiler +deduces, never one this package guessed. A variable may not change type between +assignments. The one exception to `auto` is a variable assigned inside a nested +block: Python scopes it to the whole function, so it is hoisted to function +scope and therefore has to be declared with a type that can be determined up +front. Annotate it (`total: 'double' = 0.0`) if it cannot be inferred. Such a +variable also has to be assigned before the block that reads it -- Python would +raise `UnboundLocalError` if the block never ran, and a zero that looks like a +result is the wrong way to translate that. + +Element assignment (`a[0] = x`) is only allowed on an array the function itself +built, as by `np.array([...])`. In Python `w = v` and `w = v[1:]` make `w` refer +to the same data as `v`, while the translation copies, so writing through one of +those would mean different things in the two languages. + +The translation reads the callable's Python source, so a callable whose source +cannot be recovered -- typed at the plain `python` prompt, passed to `python +-c`, piped in on standard input, or built by `exec()` -- cannot be translated. +Functions in a file and in a notebook cell both work. Not supported, and refused with a message pointing at the line: comprehensions, -lambdas inside the body, dicts, sets, tuples, strings, `try`, `with`, `global`, -generators, f-strings, `*args`, multi-dimensional indexing, and `np.random` -(a C++ generator would not reproduce numpy's stream, and silently different -random numbers are worse than an error). +lambdas inside the body, dicts, sets, tuples, tuple unpacking, strings, `try`, +`with`, `global`, generators, f-strings, `*args`, multi-dimensional indexing, +array allocation such as `np.zeros`, calls to plain Python functions, and +`np.random` (a C++ generator would not reproduce numpy's stream, and silently +different random numbers are worse than an error). diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py index d0285611720d4..657cb7201690a 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py @@ -17,14 +17,10 @@ anything cling knows about (including user classes), and needs no third-party dependency. -Two backends are available and produce the same C++ for the same input: - -``ast`` - Walks the Python syntax tree. Handles control flow. - -``trace`` - Runs the callable once with symbolic arguments and records the operations. - Cannot see control flow, and says so rather than guessing. +The translation walks the Python syntax tree. Because the argument types are +known -- RDataFrame knows its column types, and the decorator takes them +explicitly -- forward type inference through the tree is enough to choose the +right C++ for every operation, including control flow. Usage:: @@ -44,11 +40,9 @@ def pypow(x, y): from . import emit, support from .ast_backend import CPP_KEYWORDS, AstTranspiler from .errors import PyDeclareError -from .trace_backend import trace as _trace __all__ = ["Declare", "PyDeclareError", "declare"] -BACKENDS = ("ast", "trace") DEFAULT_NAMESPACE = "Py" #: (namespace, name) -> generated code, so that re-running a script in the same @@ -56,13 +50,6 @@ def pypow(x, y): _DECLARED = {} -def _default_backend(): - backend = os.environ.get("ROOT_PYDECLARE_BACKEND", "ast").strip().lower() - if backend not in BACKENDS: - raise PyDeclareError("ROOT_PYDECLARE_BACKEND is '{}'; expected one of {}".format(backend, ", ".join(BACKENDS))) - return backend - - def _sanitize(name): if name in CPP_KEYWORDS or name.startswith("PyD"): return name + "_" @@ -104,16 +91,12 @@ def _signature_names(func, n_expected): return names -def declare(func, input_types, return_type=None, name=None, namespace=DEFAULT_NAMESPACE, backend=None, unroll=False): +def declare(func, input_types, return_type=None, name=None, namespace=DEFAULT_NAMESPACE): """Translate a Python callable to C++ and declare it to cling. Returns the original callable, with the generated code and some metadata attached as attributes. """ - backend = (backend or _default_backend()).strip().lower() - if backend not in BACKENDS: - raise PyDeclareError("Unknown backend '{}'; expected one of {}".format(backend, ", ".join(BACKENDS))) - raw_types = list(input_types or []) param_names = _signature_names(func, len(raw_types)) parsed = [ct.parse_type(t) for t in raw_types] @@ -139,18 +122,15 @@ def declare(func, input_types, return_type=None, name=None, namespace=DEFAULT_NA body_names.append(cpp_params[i]) body_types.append(t) - if backend == "ast": - transpiler = AstTranspiler(func, param_names, body_types, declared_return, body_names) - body, deduced = transpiler.translate() - else: - body, deduced = _trace(func, param_names, body_types, declared_return, body_names, unroll=unroll) + transpiler = AstTranspiler(func, param_names, body_types, declared_return, body_names) + body, deduced = transpiler.translate() cpp_return = (declared_return or deduced).cpp() if (declared_return or deduced) is not None else "auto" func_name = name or getattr(func, "__name__", None) if not func_name or func_name == "": raise PyDeclareError("Anonymous callables need an explicit name=... argument") - code = _render(namespace, func_name, cpp_return, declarations, adapters, body, func, backend) + code = _render(namespace, func_name, cpp_return, declarations, adapters, body, func) key = (namespace, func_name) previous = _DECLARED.get(key) @@ -170,11 +150,10 @@ def declare(func, input_types, return_type=None, name=None, namespace=DEFAULT_NA func.__cpp_wrapper__ = code func.__pydeclare_cpp_name__ = "{}::{}".format(namespace, func_name) func.__pydeclare_return_type__ = declared_return or deduced - func.__pydeclare_backend__ = backend return func -def _render(namespace, func_name, cpp_return, declarations, adapters, body, func, backend): +def _render(namespace, func_name, cpp_return, declarations, adapters, body, func): origin = "" try: origin = " ({}:{})".format(os.path.basename(inspect.getsourcefile(func) or "?"), func.__code__.co_firstlineno) @@ -188,14 +167,14 @@ def _render(namespace, func_name, cpp_return, declarations, adapters, body, func "/// Generated from the Python callable '{}'{} by the ROOT".format( getattr(func, "__name__", func_name), origin ), - "/// Python-to-C++ transpiler, {} backend.".format(backend), + "/// Python-to-C++ transpiler.", "{} {}({})".format(cpp_return, func_name, ", ".join(declarations)), "{", ] return "\n".join(header + adapters + [body, "}", "", "}} // namespace {}".format(namespace)]) + "\n" -def Declare(input_types=None, return_type=None, name=None, namespace=DEFAULT_NAMESPACE, backend=None, unroll=False): +def Declare(input_types=None, return_type=None, name=None, namespace=DEFAULT_NAMESPACE): """Decorator making a Python callable available in C++, by translation. Arguments mirror ``ROOT.Numba.Declare``: @@ -209,11 +188,6 @@ def Declare(input_types=None, return_type=None, name=None, namespace=DEFAULT_NAM The name of the generated C++ function; defaults to the Python name. namespace The C++ namespace to declare it in; defaults to ``Py``. - backend - ``"ast"`` or ``"trace"``; defaults to $ROOT_PYDECLARE_BACKEND or ast. - unroll - Tracing backend only: accept Python-level control flow, evaluating it - once at declaration time. """ def inner(func): @@ -223,30 +197,23 @@ def inner(func): return_type=return_type, name=name, namespace=namespace, - backend=backend, - unroll=unroll, ) return inner def _numba_declare_dispatch(input_types, return_type=None, name=None, **kwargs): - """``ROOT.Numba.Declare``, routed to whichever backend is selected. - - ``ROOT_NUMBA_DECLARE_BACKEND`` picks the implementation: ``numba`` (the - default, the original implementation), ``ast`` or ``trace``. The generated - function always lands in the ``Numba`` C++ namespace, so existing code that - says ``"Numba::myfunc(x)"`` keeps working whichever backend is in use. + """``ROOT.Numba.Declare``, routed to the implementation selected by + ``$ROOT_NUMBA_DECLARE_BACKEND``: ``numba`` (the default, the original + implementation) or ``ast``. The generated function always lands in the + ``Numba`` C++ namespace, so existing code that says ``"Numba::myfunc(x)"`` + keeps working with either implementation. """ backend = os.environ.get("ROOT_NUMBA_DECLARE_BACKEND", "numba").strip().lower() if backend in ("", "numba"): from .._numbadeclare import _NumbaDeclareDecorator return _NumbaDeclareDecorator(input_types, return_type, name) - if backend not in BACKENDS: - raise PyDeclareError( - "ROOT_NUMBA_DECLARE_BACKEND is '{}'; expected 'numba', {}".format( - backend, " or ".join(repr(b) for b in BACKENDS) - ) - ) - return Declare(input_types, return_type, name, namespace="Numba", backend=backend, **kwargs) + if backend != "ast": + raise PyDeclareError("ROOT_NUMBA_DECLARE_BACKEND is '{}'; expected 'numba' or 'ast'".format(backend)) + return Declare(input_types, return_type, name, namespace="Numba", **kwargs) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py index c8b27997f9ae0..f4c19cc106a4d 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py @@ -8,15 +8,16 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ -"""The AST-traversal backend. +"""The Python-to-C++ translation. Walks the Python syntax tree of the decorated callable and emits C++. Because the argument types are known -- RDataFrame knows its column types, and the decorator takes them explicitly -- forward type inference through the tree is enough to choose the right lowering for every operation. -Compared to the tracing backend this one sees the *source*, so it can translate -control flow: if/elif/else, for, while, break/continue and early returns. +Working from the source rather than from a traced execution is what makes +control flow translatable: if/elif/else, for, while, break/continue and early +returns all survive the trip. """ import ast @@ -115,6 +116,8 @@ def __init__(self, func, param_names, param_types, declared_return, cpp_param_na self.hoisted = {} # python name -> (cpp name, CppType) self.collected = {} # python name -> CppType, filled by the first pass self.collected_depth = {} # python name -> set of block depths it is assigned at + self.read_at_top = set() # names read at function-block level + self.fresh_arrays = set() # locals holding an array that is a copy in Python too self.return_types = [] self.loop_depth = 0 self.globals_cache = None @@ -158,6 +161,22 @@ def translate(self): self.lines, self.indent = saved hoist_names = {n for n, depths in self.collected_depth.items() if any(d > 0 for d in depths)} + # A parameter is already a function-scope variable; re-declaring it + # would be a redefinition in C++. + hoist_names -= set(self.param_names) + + for name in sorted(hoist_names): + if 0 in self.collected_depth[name] or name not in self.read_at_top: + continue + # Python would raise UnboundLocalError here if the block never ran. + # C++ has no such state, so the alternative to refusing would be to + # hand back a zero that looks like a result. + raise PyDeclareError( + "The variable '{}' is only assigned inside a nested block but is read outside it. " + "If the block does not run, Python raises UnboundLocalError, which cannot be " + "reproduced in C++.\n" + " Assign it before the block, for example \"{} = 0.0\".".format(name, name) + ) # Pass 2: the real one. self.flat_scope = False @@ -209,9 +228,9 @@ def _parse(self): src = inspect.getsource(self.func) except (OSError, TypeError): raise PyDeclareError( - "Cannot read the source of the callable. The AST backend needs the Python source; " - "callables defined in the interpreter prompt or in an exec() string cannot be " - "translated. Use the tracing backend, or move the function into a file." + "Cannot read the source of the callable. The translation reads the Python source, " + "so callables defined at an interactive prompt or built by exec() cannot be " + "translated. Define the function in a file, or in a notebook cell." ) src = textwrap.dedent(src) try: @@ -333,6 +352,36 @@ def st_AugAssign(self, node): result = emit.binop(op, current, self.expr(node.value), self.fail_at(node)) self._assign_to(node.target, result, node) + def _is_fresh_array(self, node): + """True if this assignment's right-hand side builds a new array. + + np.array([...]) and .astype(...) copy in numpy as well as here. A bare + name, a slice or a mask produces a view in numpy and a copy in C++, so + those are not fresh and may not be written through. + """ + value = getattr(node, "value", None) + if not isinstance(value, ast.Call): + return False + func = value.func + if isinstance(func, ast.Attribute): + return func.attr in ("array", "astype", "copy", "zeros", "ones", "full", "empty") + return False + + def _check_mutable_target(self, target, node): + base = target.value + if isinstance(base, ast.Name) and base.id in self.fresh_arrays: + return + what = "'{}'".format(base.id) if isinstance(base, ast.Name) else "this array" + raise_unsupported( + self.func, + node, + "Cannot assign to an element of {}".format(what), + hint="In Python, 'w = v' and 'w = v[1:]' make w refer to the same data as v, while the " + "translation copies, so an element assignment would mean different things in the two " + "languages. Only an array built in the function, as by np.array([...]), can be written " + "to element by element.", + ) + def _record(self, name, type_): """Remember an assignment for the hoisting pass.""" self.collected_depth.setdefault(name, set()).add(self.depth) @@ -352,6 +401,7 @@ def _converted(self, value, target_type): def _assign_to(self, target, value, node): if isinstance(target, ast.Subscript): + self._check_mutable_target(target, node) obj = self.expr(target.value) index = self.expr(target.slice) slot = emit.subscript(obj, index, self.fail_at(node)) @@ -366,6 +416,25 @@ def _assign_to(self, target, value, node): cpp = self.cpp_name(name) self._record(name, value.type) + if name in self.param_names and value.type.is_container: + raise_unsupported( + self.func, + node, + "Cannot assign to the array parameter '{}'".format(name), + hint="Array parameters are passed by const reference, so the caller's array is " + "never modified. Assign to a new variable instead.", + ) + + if value.type.is_container: + # Track whether this name holds an array that is a fresh copy in + # Python too, so that element assignment through it means the same + # thing in both languages. numpy aliases on 'w = v' and on a + # slice; the generated C++ copies, so those may not be mutated. + if self._is_fresh_array(node): + self.fresh_arrays.add(name) + else: + self.fresh_arrays.discard(name) + if name in self.hoisted: declared = self.hoisted[name][1] self.write("{} = {};".format(cpp, self._converted(value, declared))) @@ -444,8 +513,13 @@ def st_For(self, node): ) elem = iterable.type.scalar() self._record(name, elem) - self.write("for (auto {} : {}) {{".format(cpp, iterable.code)) - self._loop_body(node, name, cpp, elem) + if name in self.hoisted: + item = self.tmp("item") + self.write("for (auto &&{} : {}) {{".format(item, iterable.code)) + self._loop_body(node, name, cpp, elem, preamble="{} = {};".format(cpp, item)) + else: + self.write("for (auto {} : {}) {{".format(cpp, iterable.code)) + self._loop_body(node, name, cpp, elem) def _is_range(self, func): return isinstance(func, ast.Name) and func.id == "range" and self.scope.lookup("range") is None @@ -480,21 +554,40 @@ def _for_range(self, node, name, cpp): if abs(step_value) != 1 else ("++" + cpp if step_value > 0 else "--" + cpp) ) - self.write( - "for (long {} = static_cast({}); {} {} {}; {}) {{".format(cpp, start.code, cpp, cmp_op, bound, incr) - ) + # A variable assigned inside a nested block lives at function scope; if + # the loop declared its own, it would shadow that one and Python's + # "the loop variable is still there afterwards" would not hold. The + # counter stays separate so that the visible variable keeps the last + # value the loop ran with rather than the bound it stopped at. self._record(name, ct.LONG_T) - self._loop_body(node, name, cpp, ct.LONG_T) + if name in self.hoisted: + counter = self.tmp("i") + incr = incr.replace(cpp, counter) + self.write( + "for (long {} = static_cast({}); {} {} {}; {}) {{".format( + counter, start.code, counter, cmp_op, bound, incr + ) + ) + self._loop_body(node, name, cpp, ct.LONG_T, preamble="{} = {};".format(cpp, counter)) + else: + self.write( + "for (long {} = static_cast({}); {} {} {}; {}) {{".format( + cpp, start.code, cpp, cmp_op, bound, incr + ) + ) + self._loop_body(node, name, cpp, ct.LONG_T) self.indent -= 1 self.write("}") - def _loop_body(self, node, name, cpp, elem_type): + def _loop_body(self, node, name, cpp, elem_type, preamble=None): self.indent += 1 self.depth += 1 self.loop_depth += 1 if not self.flat_scope: self.scope = _Scope(self.scope) self.scope.names[name] = emit.Value(cpp, elem_type) + if preamble is not None: + self.write(preamble) for stmt in node.body: self.exec_stmt(stmt) if not self.flat_scope: @@ -550,6 +643,8 @@ def ex_Constant(self, node): return emit.literal(node.value, self.fail_at(node)) def ex_Name(self, node): + if self.depth == 0: + self.read_at_top.add(node.id) local = self.scope.lookup(node.id) if local is not None: return local @@ -751,7 +846,7 @@ def _math_call(self, ref, args, node): raise_unsupported(self.func, node, "'{}' expects an array".format(ref.path)) return emit.array_method(args[0], _REDUCTION_METHOD[emit.FREE_REDUCTIONS[name]], [], fail) if name == "where" and len(args) == 3: - return emit.ternary(args[0], args[1], args[2], fail) + return emit.ternary(args[0], args[1], args[2], fail, elementwise=True) if name in ("sqrt", "fabs") and len(args) == 1: return emit.unary_function(emit.UNARY_FUNCTIONS[name], args[0], fail) raise_unsupported( @@ -810,9 +905,25 @@ def ex_List(self, node): "would need a dynamic container.", ) - ex_Tuple = ex_List - ex_Dict = ex_List - ex_Set = ex_List + def ex_Tuple(self, node): + raise_unsupported( + self.func, + node, + "Tuple literals are not supported", + hint="A translated function has exactly one value of one C++ type; there is nothing " + "to translate a tuple to.", + ) + + def ex_Dict(self, node): + raise_unsupported( + self.func, + node, + "Dict literals are not supported", + hint="Use an array indexed by an integer, or a chain of if statements.", + ) + + def ex_Set(self, node): + raise_unsupported(self.func, node, "Set literals are not supported") def ex_ListComp(self, node): raise_unsupported( @@ -956,7 +1067,14 @@ def _is_static(self, node): return True def _callee_is_safe(self, func_node): - """True if this callee is a numpy/math function or a pure builtin.""" + """True if this callee is a pure numpy/math function or a pure builtin. + + Purity is what matters, and it is not a property of the package: the + whole point of numpy.random is to return something different every + time, and freezing one draw into the source would silently give every + entry the same random number. Nothing under the impure submodules + below may run, and neither may anything that touches the filesystem. + """ if isinstance(func_node, ast.Name) and func_node.id not in self._get_globals(): return func_node.id in PURE_BUILTINS try: @@ -964,8 +1082,10 @@ def _callee_is_safe(self, func_node): obj = eval(code, dict(self._get_globals())) # noqa: S307 except Exception: return False - module = (getattr(obj, "__module__", "") or "").split(".")[0] - if module in ("numpy", "math"): + module = (getattr(obj, "__module__", "") or "") or "" + if module.split(".")[0] in ("numpy", "math"): + if module.split(".")[0] == "numpy" and not self._numpy_callable_is_pure(obj, module): + return False return True # A bound method of a numpy object, as in np.array([...]).astype(...) receiver = getattr(obj, "__self__", None) @@ -973,6 +1093,13 @@ def _callee_is_safe(self, func_node): return (type(receiver).__module__ or "").split(".")[0] == "numpy" return False + @staticmethod + def _numpy_callable_is_pure(obj, module): + for impure in _IMPURE_NUMPY_MODULES: + if module == impure or module.startswith(impure + "."): + return False + return getattr(obj, "__name__", "") not in _IMPURE_NUMPY_FUNCTIONS + def _fold_safe(self, node): """True if evaluating this expression at declaration time is harmless. @@ -1003,6 +1130,28 @@ def _try_fold(self, node): return emit.constant_value(obj, self.fail_at(node)) +#: numpy submodules whose whole point is to return something different on +#: every call, or to read the outside world. Nothing in them may be folded. +_IMPURE_NUMPY_MODULES = ("numpy.random", "numpy.random.mtrand", "numpy.testing") + +#: numpy functions that touch the filesystem. +_IMPURE_NUMPY_FUNCTIONS = frozenset( + [ + "load", + "loadtxt", + "save", + "savetxt", + "savez", + "savez_compressed", + "fromfile", + "tofile", + "genfromtxt", + "memmap", + "fromregex", + ] +) + + class _BoundMethod: def __init__(self, obj, name): self.obj = obj diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py index 94e61d26ffd30..0c712559c5ffc 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py @@ -10,11 +10,11 @@ """Lowering of Python operations to C++ expressions. -This module is shared by both backends: the AST walker and the tracer both -reduce to calls into the functions here, so the two produce identical C++ for -identical Python. A backend contributes only the *discovery* of the operations -(walking a syntax tree versus recording overloaded operators); the semantics of -each individual operation live here, exactly once. +The syntax tree walk in ast_backend discovers *which* operations a callable +performs; this module decides what each one becomes in C++. Keeping the two +apart means every semantic decision -- when a division needs Python's rules, +when '|' is logical rather than bitwise, when an index has to wrap around -- +is written down exactly once, and can be read without following a tree walk. """ from . import cpptypes as ct @@ -173,7 +173,10 @@ def binop(op, a, b, fail): if op == "//": _check_numeric(op, a, b, fail) - return _pyd("FloorDiv", [a, b], ct.promote(a.type, b.type)) + t = ct.promote(a.type, b.type) + if t.is_integral: + t = ct.with_scalar(t, ct.int_op_type(a.type, b.type)) + return _pyd("FloorDiv", [a, b], t) if op == "%": _check_numeric(op, a, b, fail) @@ -181,13 +184,16 @@ def binop(op, a, b, fail): # For unsigned integers C++ '%' already agrees with Python's. if a.type.is_integral and b.type.is_integral and not a.type.is_signed and not b.type.is_signed: return _binary("%", a, b, t) + if t.is_integral: + t = ct.with_scalar(t, ct.int_op_type(a.type, b.type)) return _pyd("Mod", [a, b], t) if op == "**": _check_numeric(op, a, b, fail) t = ct.promote(a.type, b.type) if t.is_integral: - return _pyd("Pow", [a, b], t) + # PyD::Pow accumulates in a wide integer, as Python does. + return _pyd("Pow", [a, b], ct.with_scalar(t, ct.int_op_type(a.type, ct.LLONG_T))) return _pyd("Pow", [a, b], ct.with_scalar(t, DOUBLE_T)) if op in ("|", "&", "^"): @@ -216,17 +222,28 @@ def binop(op, a, b, fail): fail("Binary operator '{}' is not supported".format(op)) +def _unary_code(op, a): + """'-' followed by '-x' would fuse into the decrement token.""" + inner = a.paren(PREC_UNARY) + sep = " " if inner.startswith(op) else "" + return "{}{}{}".format(op, sep, inner) + + def unaryop(op, a, fail): """Lower a Python unary operator to C++.""" if op == "+": - return Value("+{}".format(a.paren(PREC_UNARY)), a.type, PREC_UNARY) + return Value(_unary_code("+", a), a.type, PREC_UNARY) if op == "-": if a.type.kind == ct.OPAQUE: fail("Unary '-' is not supported for values of C++ type '{}'".format(a.type.name)) t = a.type if t.is_bool: t = ct.with_scalar(t, ct.INT_T) - return Value("-{}".format(a.paren(PREC_UNARY)), t, PREC_UNARY) + elif t.is_integral and not t.is_signed: + # Negating an unsigned value wraps around in C++; Python has no + # unsigned integers and simply produces a negative number. + return _pyd("Neg", [a], ct.with_scalar(t, ct.int_op_type(t, ct.LLONG_T))) + return Value(_unary_code("-", a), t, PREC_UNARY) if op == "not": return Value("!{}".format(truth(a, fail).paren(PREC_UNARY)), ct.bool_like(a.type), PREC_UNARY) if op == "~": @@ -270,27 +287,84 @@ def compare(op, a, b, fail): if a.type.kind == ct.OPAQUE or b.type.kind == ct.OPAQUE: # Let C++ resolve the operator on the class; we cannot infer the result. return _binary(op, a, b, UNKNOWN_T) + a, b = _balance_signedness(a, b) + # A comparison used as an operand of another comparison has to be + # parenthesised: C++ rejects the chained spelling 'a >= b > c'. + a = _parenthesise_comparison(a) + b = _parenthesise_comparison(b) return _binary(op, a, b, ct.bool_like(ct.promote(a.type, b.type))) +def _parenthesise_comparison(v): + if v.prec in (PREC_REL, PREC_EQ): + return Value("({})".format(v.code), v.type, PREC_PRIMARY) + return v + + +def _balance_signedness(a, b): + """Compare a signed and an unsigned operand the way Python would. + + C++ converts the signed side, so -1 > 2u is true. Widening both sides to + a signed 64-bit type reproduces Python for every operand pair narrower + than 64 bits, which is all of them except (unsigned) long long itself. + """ + ta, tb = a.type, b.type + if not (ta.is_integral and tb.is_integral): + return a, b + if ta.is_container or tb.is_container: + return a, b + if ta.is_signed == tb.is_signed: + return a, b + wide = ct.int_op_type(ta, tb) + return _widen(a, wide), _widen(b, wide) + + +def _widen(v, t): + if v.type == t: + return v + return Value("static_cast<{}>({})".format(t.cpp(), v.code), t, PREC_UNARY) + + def boolop(op, values, fail): """Lower 'and'/'or'. Only defined for scalars, as in numpy.""" - cpp = "&&" if op == "and" else "||" for v in values: if v.type.is_container: fail( "'{}' is not defined for arrays".format(op), hint="Use '&' and '|' for element-wise logic, as you would with numpy.", ) - out = truth(values[0], fail) - for v in values[1:]: - out = _binary(cpp, out, truth(v, fail), BOOL_T) + if not (v.type.is_numeric or v.type.is_unknown): + fail("Values of C++ type '{}' cannot be used with '{}'".format(v.type, op)) + + if all(v.type.is_bool for v in values): + # The result is a bool either way, so keep the plain C++ spelling. + cpp = "&&" if op == "and" else "||" + out = values[0] + for v in values[1:]: + out = _binary(cpp, out, v, BOOL_T) + return out + + # Python's 'and'/'or' evaluate to one of their operands rather than to a + # bool: '0.0 or 5.0' is 5.0. The right-hand side goes in as a lambda so + # that it is only evaluated when Python would evaluate it. + fn = "And" if op == "and" else "Or" + out = values[-1] + for v in reversed(values[:-1]): + t = ct.promote(v.type, out.type) + code = "{}::{}({}, [&]() {{ return {}; }})".format(PYD, fn, v.code, out.code) + out = Value(code, t, PREC_PRIMARY) return out -def ternary(cond, a, b, fail): - """Lower a Python conditional expression.""" +def ternary(cond, a, b, fail, elementwise=False): + """Lower a Python conditional expression, or np.where when *elementwise*.""" if cond.type.is_container: + if not elementwise: + fail( + "The truth value of an array is ambiguous", + hint="'a if mask else b' is not defined for arrays in Python either. " + "Use np.where(mask, a, b) for the element-wise choice.", + ) return _pyd("Where", [cond, a, b], ct.with_scalar(cond.type, ct.promote(a.type, b.type).scalar())) t = ct.promote(a.type, b.type) code = "{} ? {} : {}".format(truth(cond, fail).paren(PREC_COND + 1), a.paren(PREC_COND + 1), b.paren(PREC_COND)) @@ -324,10 +398,9 @@ def subscript(obj, index, fail): if not (index.type.is_integral or index.type.is_unknown): fail("Array indices must be integers, got '{}'".format(index.type)) - # A non-negative literal, or an unsigned index, needs no wrap-around check. - literal_nonneg = index.code.isdigit() - if literal_nonneg or (index.type.is_integral and not index.type.is_signed): - return Value("{}[{}]".format(obj.paren(PREC_PRIMARY), index.code), elem, PREC_PRIMARY) + # PyD::Index wraps negative indices and rejects out-of-range ones, as + # Python does. Reading past the end of an event's array silently would be + # a poor trade for one comparison. return _pyd("Index", [obj, index], elem) @@ -341,6 +414,14 @@ def slice_(obj, start, stop, step, fail): elem = obj.type.scalar() out_t = ct.rvec_of(elem) + for name, part in (("start", start), ("stop", stop), ("step", step)): + if part is None: + continue + if part.type.is_container: + fail("A slice {} may not be an array".format(name)) + if not (part.type.is_integral or part.type.is_unknown): + fail("Slice indices must be integers, got '{}' for the {}".format(part.type, name)) + # v[::-1] is a plain reverse; emit the readable form. if start is None and stop is None and step is not None and step.code == "-1": return call("ROOT::VecOps::Reverse", [obj], out_t) @@ -366,8 +447,8 @@ def slice_(obj, start, stop, step, fail): #: numpy-style reductions available as methods on arrays. ARRAY_METHODS = { - "sum": ("Sum", "scalar"), - "prod": ("Prod", "scalar"), + "sum": ("Sum", "accumulate"), + "prod": ("Prod", "accumulate"), "mean": ("Mean", "double"), "std": ("Std", "double"), "min": ("Min", "scalar"), @@ -500,8 +581,8 @@ def _reduction_type(kind, obj): if kind == "bool": return BOOL_T elem = obj.type.scalar() - if elem.is_bool: - return LONG_T + if kind == "accumulate": + return ct.acc_type(elem) return elem diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py index 580a02fab76b1..656360c6dfa46 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/errors.py @@ -8,7 +8,7 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ -"""Errors raised by the Python-to-C++ transpiler backends. +"""Errors raised by the Python-to-C++ transpiler. The guiding principle of this package is that anything outside the supported subset must fail *loudly*, at declaration time, with a message that points at @@ -24,7 +24,7 @@ class PyDeclareError(Exception): """Raised when a Python callable cannot be translated to C++. The message carries the location in the user's Python source whenever the - backend was able to determine it. + translation was able to determine it. """ diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py deleted file mode 100644 index 5d68aab26d329..0000000000000 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/trace_backend.py +++ /dev/null @@ -1,700 +0,0 @@ -# Author: Jonas Rembser CERN 08/2026 - -################################################################################ -# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. # -# All rights reserved. # -# # -# For the licensing terms see $ROOTSYS/LICENSE. # -# For the list of contributors see $ROOTSYS/README/CREDITS. # -################################################################################ - -"""The tracing backend. - -Instead of reading the Python source, this backend *runs* the callable once -with symbolic arguments. Every operator, method and numpy ufunc applied to a -symbol records itself, and what comes back is a DAG describing the computation, -which is then emitted as C++. - -The trade-off against the AST backend is control flow. A traced symbol has no -value, so `if x > 0:` cannot pick a branch; `__bool__` raises instead of -guessing. Python-level control flow that does *not* depend on the arguments is -a different matter -- it unrolls, which is occasionally exactly what you want -- -but since it silently changes what the generated code does relative to reading -the source, it has to be asked for explicitly with ``unroll=True``. -""" - -import ast -import inspect -import sys -import textwrap - -from . import emit -from .cpptypes import UNKNOWN_T -from .errors import PyDeclareError - -_CONTROL_FLOW_NODES = (ast.If, ast.For, ast.While, ast.Try, ast.With, ast.IfExp) - - -class _TraceError(PyDeclareError): - pass - - -def _here(func): - """Format the position inside the traced callable, for error messages.""" - import traceback - - code = getattr(func, "__code__", None) - frames = traceback.extract_stack() - for frame in reversed(frames): - if code is not None and frame.filename == code.co_filename and frame.name == code.co_name: - return '\n File "{}", line {}\n {}\n'.format(frame.filename, frame.lineno, (frame.line or "").strip()) - return "" - - -class Sym: - """A symbolic value standing in for one C++ expression. - - Nodes are built lazily: each carries a builder that turns the *reference - codes* of its children into a Value. That lets the emitter decide after - the fact whether a shared sub-expression should become a named temporary - rather than being pasted in twice. - """ - - __slots__ = ("ctx", "builder", "children", "value", "name", "uses") - - __array_priority__ = 1000.0 - - def __init__(self, ctx, builder, children, value): - self.ctx = ctx - self.builder = builder - self.children = children - self.value = value # eagerly built, for its type - self.name = None # set if this node is materialised as a temporary - self.uses = 0 - ctx.nodes.append(self) - - # -- construction helpers ---------------------------------------------- - - @property - def type(self): - return self.value.type - - def rebuild(self, child_values): - return self.builder(child_values) - - def __repr__(self): - return "Sym({!r}, {})".format(self.value.code, self.value.type) - - # -- the loud failures -------------------------------------------------- - - def _refuse(self, what, hint): - raise _TraceError( - "{} while tracing '{}'.{}\n {}".format( - what, getattr(self.ctx.func, "__name__", ""), _here(self.ctx.func), hint - ) - ) - - def __bool__(self): - self._refuse( - "A traced value was converted to a Python bool", - "This happens for 'if x', 'while x', 'not x', 'x and y', 'x or y' and 'assert x'. " - "The interpreter resolves all of them by asking the value for its truth, and a " - "symbol has none to give: the tracing backend records what one Python run does, so " - "it never sees the operation at all.\n" - " Use backend='ast', which reads the source and translates these directly. For " - "element-wise logic the numpy spelling also works in both backends: '~x' for not, " - "'&' for and, '|' for or, and np.where(cond, a, b) for a conditional.", - ) - - def __len__(self): - self._refuse( - "len() was called on a traced array", - "The length is only known at run time. Use x.size, which stays symbolic.", - ) - - def __iter__(self): - self._refuse( - "A traced array was iterated over", - "The number of elements is only known at run time. Use element-wise array " - "expressions, or backend='ast' with an explicit loop.", - ) - - def __index__(self): - self._refuse("A traced value was used as an index into a Python object", "Index the array itself instead.") - - __int__ = __index__ - - def __float__(self): - self._refuse( - "A traced value was converted to a Python float", - "Arithmetic stays symbolic; use np.* functions rather than math.* ones, which force " - "a conversion to a Python number.", - ) - - def __hash__(self): - return id(self) - - # -- operators ---------------------------------------------------------- - - def _binop(self, op, other, swap=False): - a, b = (self.ctx.wrap(other), self) if swap else (self, self.ctx.wrap(other)) - return self.ctx.node(lambda vals: emit.binop(op, vals[0], vals[1], self.ctx.fail), [a, b]) - - def _cmp(self, op, other): - a, b = self, self.ctx.wrap(other) - return self.ctx.node(lambda vals: emit.compare(op, vals[0], vals[1], self.ctx.fail), [a, b]) - - -def _install_operators(): - binary = { - "add": "+", - "sub": "-", - "mul": "*", - "truediv": "/", - "floordiv": "//", - "mod": "%", - "pow": "**", - "lshift": "<<", - "rshift": ">>", - "and": "&", - "or": "|", - "xor": "^", - } - for pyname, op in binary.items(): - - def make(op=op, swap=False): - def method(self, other): - return self._binop(op, other, swap) - - return method - - setattr(Sym, "__{}__".format(pyname), make(op, False)) - setattr(Sym, "__r{}__".format(pyname), make(op, True)) - - for pyname, op in {"lt": "<", "le": "<=", "gt": ">", "ge": ">=", "eq": "==", "ne": "!="}.items(): - - def make_cmp(op=op): - def method(self, other): - return self._cmp(op, other) - - return method - - setattr(Sym, "__{}__".format(pyname), make_cmp(op)) - - for pyname, op in {"neg": "-", "pos": "+", "invert": "~"}.items(): - - def make_un(op=op): - def method(self): - return self.ctx.node(lambda vals: emit.unaryop(op, vals[0], self.ctx.fail), [self]) - - return method - - setattr(Sym, "__{}__".format(pyname), make_un(op)) - - def _abs(self): - return self.ctx.node(lambda vals: emit.unary_function("Abs", vals[0], self.ctx.fail), [self]) - - Sym.__abs__ = _abs - - -_install_operators() - - -# --- indexing --------------------------------------------------------------- - - -def _getitem(self, key): - ctx = self.ctx - if isinstance(key, slice): - parts = [key.start, key.stop, key.step] - syms = [None if p is None else ctx.wrap(p) for p in parts] - children = [self] + [s for s in syms if s is not None] - - def build(vals): - it = iter(vals[1:]) - start, stop, step = (next(it) if s is not None else None for s in syms) - return emit.slice_(vals[0], start, stop, step, ctx.fail) - - return ctx.node(build, children) - index = ctx.wrap(key) - return ctx.node(lambda vals: emit.subscript(vals[0], vals[1], ctx.fail), [self, index]) - - -Sym.__getitem__ = _getitem - - -def _setitem(self, key, value): - self._refuse( - "A traced array was assigned into", - "The tracing backend builds a pure expression; it has nowhere to put a mutation. " - "Use backend='ast' if you need to write into an array.", - ) - - -Sym.__setitem__ = _setitem - - -# --- attributes ------------------------------------------------------------- - - -class _Method: - """A method looked up on a symbol but not yet called.""" - - def __init__(self, sym, name): - self.sym = sym - self.name = name - - def __call__(self, *args, **kwargs): - ctx = self.sym.ctx - if kwargs: - raise _TraceError( - "Keyword arguments are not supported in a call to '{}'.{}".format(self.name, _here(ctx.func)) - ) - if self.name == "astype": - if len(args) != 1: - raise _TraceError("astype() takes exactly one argument.{}".format(_here(ctx.func))) - dtype = args[0] - return ctx.node(lambda vals: emit.astype(vals[0], dtype, ctx.fail), [self.sym]) - if self.name in emit.ARRAY_METHODS and self.sym.type.is_container: - if args: - raise _TraceError("'{}' does not take arguments here.{}".format(self.name, _here(ctx.func))) - return ctx.node(lambda vals: emit.array_method(vals[0], self.name, [], ctx.fail), [self.sym]) - # Anything else is passed straight through to C++: this is how methods - # on ROOT classes such as PtEtaPhiMVector::M() are translated. - children = [self.sym] + [ctx.wrap(a) for a in args] - return ctx.node(lambda vals: emit.cpp_method(vals[0], self.name, vals[1:]), children) - - -def _getattr(self, name): - if name.startswith("__") or name in Sym.__slots__: - raise AttributeError(name) - ctx = self.ctx - if self.type.is_container: - if name == "size": - return ctx.node(lambda vals: emit.length(vals[0], ctx.fail), [self]) - if name in emit.ARRAY_METHODS: - return _Method(self, name) - if name in ("T", "shape", "ndim", "dtype", "flat", "real", "imag"): - raise _TraceError( - "Arrays have no '{}' in the supported subset; they are always one dimensional " - "arrays of numbers.{}".format(name, _here(ctx.func)) - ) - return _Method(self, name) - if self.type.is_fund: - raise _TraceError("'{}' has no attribute '{}'.{}".format(self.type, name, _here(ctx.func))) - # Opaque C++ object: could be a data member or a method. - return _MaybeMember(self, name) - - -class _MaybeMember(emit.Value): - """A member access on an opaque C++ object: a value, or a method to call.""" - - __slots__ = ("sym", "name", "_node") - - def __init__(self, sym, name): - ctx = sym.ctx - node = ctx.node(lambda vals: emit.cpp_member(vals[0], name), [sym]) - super().__init__(node.value.code, node.value.type, node.value.prec) - self.sym = sym - self.name = name - self._node = node - - def __call__(self, *args, **kwargs): - # It was a method after all; drop the member node and build a call. - ctx = self.sym.ctx - ctx.discard(self._node) - return _Method(self.sym, self.name)(*args, **kwargs) - - -Sym.__getattr__ = _getattr - - -# --- numpy interoperability ------------------------------------------------- - -_UFUNC_BINOPS = { - "add": "+", - "subtract": "-", - "multiply": "*", - "true_divide": "/", - "divide": "/", - "floor_divide": "//", - "remainder": "%", - "mod": "%", - "power": "**", - "float_power": "**", - "left_shift": "<<", - "right_shift": ">>", - "bitwise_and": "&", - "bitwise_or": "|", - "bitwise_xor": "^", - "logical_and": "&", - "logical_or": "|", - "logical_xor": "^", -} -_UFUNC_COMPARES = { - "less": "<", - "less_equal": "<=", - "greater": ">", - "greater_equal": ">=", - "equal": "==", - "not_equal": "!=", -} -_UFUNC_UNARY = {"negative": "-", "positive": "+", "invert": "~", "logical_not": "not"} - - -def _array_ufunc(self, ufunc, method, *inputs, **kwargs): - ctx = self.ctx - name = getattr(ufunc, "__name__", "") - if method != "__call__" or kwargs.get("out") is not None: - raise _TraceError("numpy's '{}' method of ufunc '{}' is not supported.{}".format(method, name, _here(ctx.func))) - args = [ctx.wrap(a) for a in inputs] - - if name in _UFUNC_UNARY and len(args) == 1: - op = _UFUNC_UNARY[name] - return ctx.node(lambda vals: emit.unaryop(op, vals[0], ctx.fail), args) - if name in emit.UNARY_FUNCTIONS and len(args) == 1: - fn = emit.UNARY_FUNCTIONS[name] - return ctx.node(lambda vals: emit.unary_function(fn, vals[0], ctx.fail), args) - if name in _UFUNC_BINOPS and len(args) == 2: - op = _UFUNC_BINOPS[name] - return ctx.node(lambda vals: emit.binop(op, vals[0], vals[1], ctx.fail), args) - if name in _UFUNC_COMPARES and len(args) == 2: - op = _UFUNC_COMPARES[name] - return ctx.node(lambda vals: emit.compare(op, vals[0], vals[1], ctx.fail), args) - if name in emit.BINARY_FUNCTIONS and len(args) == 2: - fn = emit.BINARY_FUNCTIONS[name] - return ctx.node(lambda vals: emit.binary_function(fn, vals[0], vals[1], ctx.fail), args) - raise _TraceError( - "The numpy function '{}' is not available in the supported subset.{}".format(name, _here(ctx.func)) - ) - - -def _array_function(self, func, types, args, kwargs): - ctx = self.ctx - name = getattr(func, "__name__", "") - if kwargs: - raise _TraceError("Keyword arguments to numpy's '{}' are not supported.{}".format(name, _here(ctx.func))) - wrapped = [ctx.wrap(a) for a in args] - if name == "where" and len(wrapped) == 3: - return ctx.node(lambda vals: emit.ternary(vals[0], vals[1], vals[2], ctx.fail), wrapped) - if name in emit.FREE_REDUCTIONS and len(wrapped) == 1: - method = _REDUCTION_METHOD[emit.FREE_REDUCTIONS[name]] - return ctx.node(lambda vals: emit.array_method(vals[0], method, [], ctx.fail), wrapped) - raise _TraceError( - "The numpy function '{}' is not available in the supported subset.{}".format(name, _here(ctx.func)) - ) - - -_REDUCTION_METHOD = {fn: name for name, (fn, _) in emit.ARRAY_METHODS.items()} - -Sym.__array_ufunc__ = _array_ufunc -Sym.__array_function__ = _array_function - - -# --- tracer-aware builtins -------------------------------------------------- -# -# float(x), len(x) and friends are resolved by the interpreter to a call that -# must return a real Python object: float() insists on a float, len() on an int. -# A symbol cannot satisfy them. They can, however, be shadowed: the traced -# callable is re-created with a __builtins__ mapping in which these names point -# at tracer-aware versions, so the ordinary Python spelling keeps working. -# -# The operations the interpreter routes through __bool__ -- 'not x', 'x and y', -# 'if x' -- have no such hook, and are reported instead. - - -def _traced_builtin(name): - import builtins - - original = getattr(builtins, name) - - def wrapper(*args): - syms = [a for a in args if isinstance(a, Sym)] - if not syms: - return original(*args) - ctx = syms[0].ctx - wrapped = [ctx.wrap(a) for a in args] - - def build(vals): - if name in ("float", "int", "bool"): - return emit.cast(name, vals[0], ctx.fail) - if name == "len": - return emit.length(vals[0], ctx.fail) - if name == "abs": - return emit.unary_function("Abs", vals[0], ctx.fail) - if name == "round": - return emit.unary_function("Round", vals[0], ctx.fail) - if name == "sum": - return emit.array_method(vals[0], "sum", [], ctx.fail) - if name == "pow": - return emit.binop("**", vals[0], vals[1], ctx.fail) - if name in ("min", "max"): - if len(vals) == 1: - return emit.array_method(vals[0], name, [], ctx.fail) - return emit.binary_function("Maximum" if name == "max" else "Minimum", vals[0], vals[1], ctx.fail) - ctx.fail("The builtin '{}' is not available in the supported subset".format(name)) - - if name in ("float", "int", "bool", "len", "abs", "round", "sum") and len(wrapped) != 1: - ctx.fail("{}() takes exactly one argument here".format(name)) - if name == "pow" and len(wrapped) != 2: - ctx.fail("pow() takes exactly two arguments here") - if name in ("min", "max") and len(wrapped) not in (1, 2): - ctx.fail("{}() takes one or two arguments here".format(name)) - return ctx.node(build, wrapped) - - wrapper.__name__ = name - return wrapper - - -class _NumpyProxy: - """Stands in for the numpy module while tracing. - - Only np.array needs intercepting: unlike the ufuncs and the functions - covered by __array_function__, it has no dispatch protocol, so a list - containing symbols would silently become an object array. - """ - - def __init__(self, module): - object.__setattr__(self, "_module", module) - - def __getattr__(self, name): - if name == "array": - return _traced_np_array - return getattr(object.__getattribute__(self, "_module"), name) - - -def _traced_np_array(obj, dtype=None): - import numpy as np - - values = list(obj) if isinstance(obj, (list, tuple)) else [] - syms = [v for v in values if isinstance(v, Sym)] - if not syms: - return np.array(obj) if dtype is None else np.array(obj, dtype=dtype) - ctx = syms[0].ctx - children = [ctx.wrap(v) for v in values] - - def build(vals): - built = emit.array_from_elements(vals, ctx.fail) - return built if dtype is None else emit.astype(built, dtype, ctx.fail) - - return ctx.node(build, children) - - -class _CppProxy: - """Stands in for a C++ namespace, class or function while tracing. - - Calling it with a symbol records a C++ call; calling it with ordinary - Python values does the real thing, so a traced callable that happens to use - cppyy for something unrelated keeps working. - """ - - def __init__(self, cpp_name, obj): - object.__setattr__(self, "_cpp_name", cpp_name) - object.__setattr__(self, "_obj", obj) - - def __getattr__(self, name): - if name.startswith("__"): - raise AttributeError(name) - obj = object.__getattribute__(self, "_obj") - prefix = object.__getattribute__(self, "_cpp_name") - try: - child = getattr(obj, name) - except AttributeError: - raise _TraceError("'{}' has no member '{}' known to cling".format(prefix or "the global namespace", name)) - if emit.cpp_entity_kind(child) is None: - raise _TraceError( - "'{}{}' is a C++ object, not a type or a function.\n" - " Only classes, namespaces and functions can be named from translated code. " - "Pass the object in as an argument instead.".format(prefix + "::" if prefix else "", name) - ) - qualified = emit.cpp_entity_name(child) or ("{}::{}".format(prefix, name) if prefix else name) - return _CppProxy(qualified, child) - - def __call__(self, *args, **kwargs): - obj = object.__getattribute__(self, "_obj") - syms = [a for a in args if isinstance(a, Sym)] - if not syms: - return obj(*args, **kwargs) - if kwargs: - raise _TraceError("Keyword arguments are not supported in a call to C++") - ctx = syms[0].ctx - name = object.__getattribute__(self, "_cpp_name") - children = [ctx.wrap(a) for a in args] - return ctx.node(lambda vals: emit.call(name, vals, UNKNOWN_T), children) - - -TRACED_BUILTINS = ("float", "int", "bool", "len", "abs", "round", "sum", "pow", "min", "max") - - -def _with_traced_builtins(func): - """A copy of func whose builtin lookups see the tracer-aware versions.""" - import builtins - import types - - if not hasattr(func, "__code__"): - return func - env = dict(vars(builtins)) - for name in TRACED_BUILTINS: - env[name] = _traced_builtin(name) - glob = dict(getattr(func, "__globals__", {}) or {}) - glob["__builtins__"] = env - numpy_module = sys.modules.get("numpy") - for name, value in list(glob.items()): - if numpy_module is not None and value is numpy_module: - glob[name] = _NumpyProxy(numpy_module) - elif emit.is_cpp_root_namespace(value): - glob[name] = _CppProxy("", value) - clone = types.FunctionType(func.__code__, glob, func.__name__, func.__defaults__, func.__closure__) - clone.__kwdefaults__ = getattr(func, "__kwdefaults__", None) - return clone - - -# --- the tracing context ---------------------------------------------------- - - -class TraceContext: - def __init__(self, func): - self.func = func - self.nodes = [] - - def fail(self, msg, hint=None): - text = "{}{}".format(msg, _here(self.func)) - if hint: - text += "\n " + hint - raise _TraceError(text) - - def node(self, builder, children): - value = builder([c.value for c in children]) - return Sym(self, builder, children, value) - - def discard(self, sym): - # Identity, not equality: '==' on a Sym builds a comparison node. - for i, node in enumerate(self.nodes): - if node is sym: - del self.nodes[i] - return - - def leaf(self, value): - return Sym(self, lambda vals: value, [], value) - - def wrap(self, obj): - """Turn a Python value into a Sym, folding constants.""" - if isinstance(obj, Sym): - return obj - if isinstance(obj, emit.Value): - return self.leaf(obj) - value = emit.constant_value(obj, self.fail) - if value is None: - self.fail( - "Cannot use a value of type '{}' in a traced expression".format(type(obj).__name__), - "Only numbers, numpy scalars and one dimensional numpy arrays of numbers can be " - "combined with traced values.", - ) - return self.leaf(value) - - -def _check_no_control_flow(func, unroll): - """Refuse to trace a function whose source contains control flow.""" - if unroll: - return - try: - src = textwrap.dedent(inspect.getsource(func)) - tree = ast.parse(src) - except (OSError, TypeError, SyntaxError): - return # no source: rely on Sym.__bool__ to catch the data-dependent cases - for node in ast.walk(tree): - if isinstance(node, _CONTROL_FLOW_NODES): - raise _TraceError( - "The callable contains a '{}' statement, which the tracing backend cannot " - "translate.\n" - ' File "{}", line {}\n' - " Tracing records the operations that one Python run performs, so a branch is " - "either invisible (it depended on the arguments) or silently baked in (it did " - "not).\n" - " Use backend='ast' to translate the control flow, or pass unroll=True to " - "accept the Python-level evaluation.".format( - type(node).__name__.lower(), - getattr(func, "__code__", None) and func.__code__.co_filename, - node.lineno + (func.__code__.co_firstlineno - 1 if hasattr(func, "__code__") else 0), - ) - ) - - -# --- emission --------------------------------------------------------------- - - -def trace(func, param_names, param_types, declared_return, cpp_param_names, unroll=False): - """Trace the callable and return (body, deduced_return_type).""" - _check_no_control_flow(func, unroll) - - ctx = TraceContext(func) - args = [ctx.leaf(emit.Value(cpp_param_names[i], param_types[i])) for i in range(len(param_names))] - - try: - result = _with_traced_builtins(func)(*args) - except PyDeclareError: - raise - except TypeError as exc: - raise _TraceError( - "Tracing '{}' failed: {}\n This usually means an operation was applied to a traced " - "value that the tracing backend does not implement.".format(getattr(func, "__name__", ""), exc) - ) - - if isinstance(result, _MaybeMember): - result = result._node - if not isinstance(result, Sym): - value = emit.constant_value(result, ctx.fail) - if value is None: - ctx.fail( - "The traced callable returned a {} rather than a value derived from its arguments".format( - type(result).__name__ - ) - ) - result = ctx.leaf(value) - - lines, root_value = _emit(ctx, result) - - if declared_return is not None: - if root_value.type == declared_return: - lines.append(" return {};".format(root_value.code)) - else: - lines.append(" return {}::Return<{}>({});".format(emit.PYD, declared_return.cpp(), root_value.code)) - deduced = declared_return - else: - lines.append(" return {};".format(root_value.code)) - deduced = None if root_value.type.is_unknown else root_value.type - - return "\n".join(lines), deduced - - -def _emit(ctx, root): - """Emit the DAG, giving a name to every sub-expression used more than once.""" - # Count uses, reachable from the root only. - order = [] - seen = set() - - def visit(node): - if id(node) in seen: - node.uses += 1 - return - seen.add(id(node)) - node.uses = 1 - for child in node.children: - visit(child) - order.append(node) - - visit(root) - - lines = [] - values = {} - for node in order: - child_values = [values[id(c)] for c in node.children] - value = node.rebuild(child_values) - # A shared sub-expression is materialised so that it is computed once. - if node.uses > 1 and node.children: - name = "_pyd_t{}".format(len(lines) + 1) - lines.append(" const auto {} = {};".format(name, value.code)) - values[id(node)] = emit.Value(name, value.type, emit.PREC_PRIMARY) - else: - values[id(node)] = value - return lines, values[id(root)] diff --git a/bindings/pyroot/pythonizations/test/CMakeLists.txt b/bindings/pyroot/pythonizations/test/CMakeLists.txt index e2bfbe22ea8ab..c803145e4f8a5 100644 --- a/bindings/pyroot/pythonizations/test/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/test/CMakeLists.txt @@ -117,14 +117,12 @@ endif() if (dataframe) # Test wrapping Python callables for use in C++ using numba ROOT_ADD_PYUNITTEST(pyroot_numbadeclare numbadeclare.py PYTHON_DEPS numba) - # The same suite, run against each implementation of Numba.Declare: the - # numba one and the two transpiler backends, which need no numba. + # The same suite, run against both implementations of Numba.Declare: + # the numba one and the transpiler, which needs no numba. ROOT_ADD_PYUNITTEST(pyroot_pydeclare_numba pydeclare.py PYTHON_DEPS numba ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=numba) ROOT_ADD_PYUNITTEST(pyroot_pydeclare_ast pydeclare.py ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=ast) - ROOT_ADD_PYUNITTEST(pyroot_pydeclare_trace pydeclare.py - ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=trace) ROOT_ADD_PYUNITTEST(pyroot_rdf_filter_pyz rdf_filter_pyz.py PYTHON_DEPS numba) ROOT_ADD_PYUNITTEST(pyroot_rdf_define_pyz rdf_define_pyz.py PYTHON_DEPS numba) # std::string_view in CPyCppyy diff --git a/bindings/pyroot/pythonizations/test/pydeclare.py b/bindings/pyroot/pythonizations/test/pydeclare.py index d05cf25ab3435..20c7fdc051ba3 100644 --- a/bindings/pyroot/pythonizations/test/pydeclare.py +++ b/bindings/pyroot/pythonizations/test/pydeclare.py @@ -1,9 +1,9 @@ """Backend-parameterised version of numbadeclare.py. -The same suite is run against every implementation of ROOT.Numba.Declare that -exists: the original numba one, and the two transpiler backends that translate -the Python callable to C++ source instead. Which one is exercised is selected -by $ROOT_NUMBA_DECLARE_BACKEND, read by the decorator itself, so apart from the +The same suite is run against both implementations of ROOT.Numba.Declare: the +original numba one, and the transpiler that translates the Python callable to +C++ source instead. Which one is exercised is selected by +$ROOT_NUMBA_DECLARE_BACKEND, read by the decorator itself, so apart from the handful of assertions that are about numba's implementation rather than about the feature, this file is the original test suite unchanged. """ @@ -117,7 +117,6 @@ def fn1(x): self.assertTrue(hasattr(fn1, "numba_func")) self.assertEqual(sys.getrefcount(fn1.numba_func), 3) else: - self.assertEqual(fn1.__pydeclare_backend__, BACKEND) self.assertEqual(fn1.__pydeclare_cpp_name__, "Numba::fn1") # Test cling integration @@ -330,21 +329,7 @@ def fn6b(x): def test_wrapper_in_b(self): """ Test wrapper with different input/output configurations - - 'not x' is the one construct in this suite that the tracing backend - cannot translate: the interpreter resolves it through __bool__, which a - symbol cannot answer. The backend has to say so rather than guess, and - that is what is checked here. """ - if BACKEND == "trace": - with self.assertRaises(Exception) as caught: - - @ROOT.Numba.Declare(["bool"], "bool") - def fn6b2_trace(x): - return not x - - self.assertIn("bool", str(caught.exception)) - return @ROOT.Numba.Declare(["bool"], "bool") def fn6b2(x): From edf7310e38c1bbea9e179ad854663370e39016fd Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 31 Aug 2026 05:21:13 +0000 Subject: [PATCH 5/8] [Python] Make ROOT.Numba.Declare translate to C++ and delete _numbadeclare.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decorator no longer compiles the callable with numba. It translates it to C++ source and gives that to cling, which is what ROOT.Py.Declare does; the spelling and the Numba C++ namespace are kept unchanged so that existing code saying "Numba::myfunc(x)" keeps working. The name is now a historical accident rather than a description, which is the price of not breaking anyone's analysis. What users get out of it: array returns stop paying for a malloc and two copies per event, declaring is an order of magnitude cheaper, methods on ROOT classes work everywhere rather than only where the experimental cppyy/numba bridge is tested, and the generated C++ can be read back off the callable as __cpp_wrapper__. numbadeclare.py is folded into pydeclare.py: the suite is the same, minus the assertions that were about numba's internals rather than the feature, plus new coverage for control flow, cppyy interop, the operators where Python and C++ disagree, and the requirement that unsupported syntax is refused with the offending source line rather than approximated. The random-data fixture in rdf_filter_pyz_helper.py used np.random inside the event loop, which the translation deliberately refuses -- no C++ generator reproduces numpy's stream, and silently different random numbers are worse than an error. It now uses gRandom directly, which is what it wanted in the first place. 🤖 Done with the help of AI --- .../pythonizations/python/CMakeLists.txt | 1 - .../pythonizations/python/ROOT/_facade.py | 9 +- .../python/ROOT/_numbadeclare.py | 522 ---------- .../python/ROOT/_pydeclare/__init__.py | 36 +- .../python/ROOT/_pythonization/_rdf_pyz.py | 23 +- .../pyroot/pythonizations/test/CMakeLists.txt | 14 +- .../pythonizations/test/numbadeclare.py | 751 -------------- .../pyroot/pythonizations/test/pydeclare.py | 940 ++++++++++++++++-- .../pythonizations/test/rdf_define_pyz.py | 24 +- .../pythonizations/test/rdf_filter_pyz.py | 9 +- .../test/rdf_filter_pyz_helper.py | 48 +- 11 files changed, 958 insertions(+), 1419 deletions(-) delete mode 100644 bindings/pyroot/pythonizations/python/ROOT/_numbadeclare.py delete mode 100644 bindings/pyroot/pythonizations/test/numbadeclare.py diff --git a/bindings/pyroot/pythonizations/python/CMakeLists.txt b/bindings/pyroot/pythonizations/python/CMakeLists.txt index bfba62dcb74ef..3ac075c70da6d 100644 --- a/bindings/pyroot/pythonizations/python/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/python/CMakeLists.txt @@ -82,7 +82,6 @@ set(py_sources ROOT/_jupyroot/magics/cppmagic.py ROOT/_jupyroot/magics/jsrootmagic.py ROOT/_jupyroot/magics/rootbrowsemagic.py - ROOT/_numbadeclare.py ROOT/_pydeclare/__init__.py ROOT/_pydeclare/ast_backend.py ROOT/_pydeclare/cpptypes.py diff --git a/bindings/pyroot/pythonizations/python/ROOT/_facade.py b/bindings/pyroot/pythonizations/python/ROOT/_facade.py index a8205b18c9e92..03b4f61e396f3 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_facade.py @@ -534,13 +534,14 @@ def TMVA(self): # Create and overload Numba namespace @property def Numba(self): - from ._pydeclare import _numba_declare_dispatch + from ._pydeclare import DeclareInNumbaNamespace self._cppyy.cppdef("namespace Numba {}") ns = self._fallback_getattr("Numba") - # The implementation behind the decorator is selected by - # $ROOT_NUMBA_DECLARE_BACKEND: 'numba' (default), 'ast' or 'trace'. - ns.Declare = staticmethod(_numba_declare_dispatch) + # Kept for backwards compatibility: the decorator no longer uses numba, + # but the spelling and the C++ namespace are unchanged. ROOT.Py.Declare + # is the same thing under a name that is not a historical accident. + ns.Declare = staticmethod(DeclareInNumbaNamespace) del type(self).Numba return ns diff --git a/bindings/pyroot/pythonizations/python/ROOT/_numbadeclare.py b/bindings/pyroot/pythonizations/python/ROOT/_numbadeclare.py deleted file mode 100644 index e945d27fd2d71..0000000000000 --- a/bindings/pyroot/pythonizations/python/ROOT/_numbadeclare.py +++ /dev/null @@ -1,522 +0,0 @@ -def _NumbaDeclareDecorator(input_types, return_type=None, name=None): - """ - Decorator for making Python callables accessible in C++ by just-in-time compilation - with numba and cling - - The decorator takes the given Python callable and just-in-time compiles (jits) - wrapper functions with the given C++ types for input and return types. Eventually, - the Python callable is accessible in the Numba namespace in C++. - - The implementation first jits with numba the Python callable. We support fundamental types as well as - ROOT::VecOps::RVec, std::vector, and std::array thereof, along with ROOT C++ classes recognized - by the cppyy Numba extension (see https://cppyy.readthedocs.io/en/latest/numba.html). - Note that you can get the jitted Python callable by the attribute numba_func. - The C++ types are converted to the respective numba types and RVecs are accessible in Python - by numpy arrays. After jitting the actual Python callable, we jit another Python wrapper, - which converts the Python signature to a C-friendly signature. The wrapper code is accessible by - the attribute __py_wrapper__. Next, the Python wrapper is given to cling to jit a C++ wrapper function, - making the original Python callable accessible in C++. The wrapper code in C++ is accessible by - the attribute __cpp_wrapper__. - - Note that the callable is fully compiled without side-effects. The numba jitting uses the nopython - option which does not allow interaction with the Python interpreter. This means that you can use - the resulting function also safely in multi-threaded environments. - """ - # Make required imports - try: - import numba as nb - except ImportError: - raise Exception("Failed to import numba") - try: - import cffi - except ImportError: - raise Exception("Failed to import cffi") - import ctypes - import re - from typing import Union - - if hasattr(nb, "version_info") and nb.version_info >= (0, 54): - pass - - CONTAINER_TYPES = { - "RVec": { - "match_pattern": r"(?:ROOT::)?(?:VecOps::)?RVec\w+|(?:ROOT::)?(?:VecOps::)?RVec<[\w\s]+>", - "cpp_name": ["ROOT::RVec", "ROOT::VecOps::RVec"], - }, - "vector": { - "match_pattern": r"(?:std::)?vector<[\w\s]+>", - "cpp_name": ["std::vector"], - }, - "array": { - "match_pattern": r"(?:std::)?array<[\w\s,<>]+>", - "cpp_name": ["std::array"], - }, - } - - def get_container_short_name(full_typename: str) -> Union[str, None]: - """ - Return the high-level container type present in a C++ type string. - - e.g. get_container_short_name("ROOT::RVecI&") -> "RVec" - get_container_short_name("std::vector") -> "std::vector" - """ - return next((c_short_name for c_short_name in CONTAINER_TYPES if c_short_name in full_typename), None) - - def is_container_type(full_typename: str) -> bool: - """ - Return True if the C++ type is a recognized container type. - - e.g. is_container_type("RVec") -> True - is_container_type("int") -> False - """ - return get_container_short_name(full_typename) is not None - - def get_container_cpp_names(full_typename: str) -> Union[str, None]: - """ - Return the full C++ name(s) for a container type. - - e.g. get_container_cpp_names("RVec") -> ["ROOT::RVec", "ROOT::VecOps::RVec"] - """ - c_short_name = get_container_short_name(full_typename) - if c_short_name is None: - return None - return CONTAINER_TYPES[c_short_name]["cpp_name"] - - def normalize_container_type(full_typename: str) -> str: - """ - Make sure all namespaces are included. - - e.g. normalize_container_type("RVecI") -> "ROOT::RVecI" - normalize_container_type("ROOT::RVec") -> "ROOT::RVec" - """ - c_short_name = get_container_short_name(full_typename) - cpp_names = get_container_cpp_names(full_typename) - - if cpp_names is None or any(full_typename.startswith(name) for name in cpp_names): - return full_typename - - # Replace short name with the first full C++ namespace-qualified name - return full_typename.replace(c_short_name, cpp_names[0]) - - # Helper functions to determine types - def get_inner_type(t: str, with_dims: bool = False) -> str: - """ - Get inner typename of a templated C++ typename - """ - try: - if "<" not in t: - # therefore, type must use a shorthand alias - typemap = { - "F": "float", - "D": "double", - "I": "int", - "U": "unsigned int", - "L": "long", - "UL": "unsigned long", - "B": "bool", - } - rvec_start = t.find("RVec") # discard a possible const modifier before "RVec" - try: - return typemap[t[rvec_start + 4 :]] # alias type characters come after "RVec" - except KeyError: - raise Exception( - "Unrecognized type {}. Valid shorthand aliases of RVec are {}".format( - t, list("RVec" + i for i in typemap.keys()) - ) - ) - g = re.match("(.*)<(.*)>", t).groups(0) - inner_type = g[1] - # Handle std::array by splitting the inner type, if requested - if "," in inner_type and not with_dims: - inner_type = inner_type.split(",")[0].strip() - return inner_type - except: # noqa E722 - raise Exception("Failed to extract template argument of type {}".format(t)) - - def map_cpp_type(t, as_ctypes=False): - """ - Get numba type object or ctype from a C++ fundamental typename - - These are the types we use to jit the Python callable. - """ - typemap = { - "float": (nb.float32, ctypes.c_float), - "double": (nb.float64, ctypes.c_double), - "int": (nb.int32, ctypes.c_int32), - "unsigned int": (nb.uint32, ctypes.c_uint32), - "long": (nb.int64, ctypes.c_int64), - "unsigned long": (nb.uint64, ctypes.c_uint64), - "bool": (nb.boolean, ctypes.c_bool), - } - if t in typemap: - return typemap[t][0] if not as_ctypes else typemap[t][1] - # fall back to void* for CFUNCTYPE - return ctypes.c_void_p if as_ctypes else t - - def get_c_signature(input_types, return_type, as_ctypes=False): - """ - Get C friendly signature as numba type objects or ctypes from C++ typenames - - We need the types to jit a Python wrapper, which can be accessed as a function pointer in C++. - """ - c_input_types = [] - for t in input_types: - if is_container_type(t): - c_input_types += [nb.types.CPointer(map_cpp_type(get_inner_type(t))), nb.int32] - else: - c_input_types.append(map_cpp_type(t, as_ctypes)) - if is_container_type(return_type): - # We return an container through pointers as part of the input arguments. Note that the - # pointer type in numba is always an int64 and is later on cast in C++ to the correct type. - # In addition, we provide the size of the data type of the array for preallocating memory of - # the returned array. - # See the Python wrapper for further information why we are using these types. - if as_ctypes: - c_return_type = ctypes.c_void_p - c_input_types += [ - ctypes.POINTER(ctypes.c_int64), # Pointer to the data (the first element of the array) - ctypes.POINTER(ctypes.c_int64), - ] # Size of the array in elements - else: - c_return_type = nb.void - c_input_types += [ - nb.types.CPointer(nb.int64), # Pointer to the data (the first element of the array) - nb.types.CPointer(nb.int64), - ] # Size of the array in elements - else: - c_return_type = map_cpp_type(return_type, as_ctypes) - return c_return_type, c_input_types - - def get_numba_signature(input_types, return_type): - """ - Get numba signature as numba type objects from C++ typenames - """ - nb_input_types = [] - for t in input_types: - if is_container_type(t): - nb_input_types.append(map_cpp_type(get_inner_type(t))[:]) - else: - nb_input_types.append(map_cpp_type(t)) - if return_type is not None: - if is_container_type(return_type): - nb_return_type = map_cpp_type(get_inner_type(return_type))[:] - else: - nb_return_type = map_cpp_type(return_type) - else: - nb_return_type = None - return nb_return_type, nb_input_types - - def add_container_input_type_ref(input_types_ref: list, const_mod: str, container_t: str) -> None: - """ - Construct the type of a container input parameter for its use in the C++ - wrapper function signature. - """ - tref = f"{const_mod}{container_t}&" - input_types_ref.append(tref) - - def add_container_func_ptr_input_type(func_ptr_input_types: list, const_mod: str, container_t: str) -> None: - """ - Construct the type of a container input parameter for its use in the cast - of the function pointer of the jitted Python wrapper. - """ - innert = get_inner_type(container_t) - if innert == "bool": - # Special treatment for bool: In numpy, bools have 1 byte - innert = "char" - - func_ptr_input_types += ["{}{}*, int".format(const_mod, innert)] - - def inner(func, input_types=input_types, return_type=return_type, name=name): - """ - Inner decorator without arguments, see outer decorator for documentation - """ - import ROOT - - # Jit the given Python callable with numba - try: - nb_return_type, nb_input_types = get_numba_signature(input_types, return_type) - if nb_return_type is not None: - nbjit = nb.jit(nb_return_type(*nb_input_types), nopython=True, inline="always")(func) - else: - nbjit = nb.jit(tuple(nb_input_types), nopython=True, inline="always")(func) - nb_return_type = nbjit.nopython_signatures[-1].return_type - except: # noqa E722 - try: - # Fallback to letting numba infer the signature - import ROOT.NumbaExt # noqa F401 - import platform - - if not (platform.system() == "Linux" and platform.machine() == "x86_64"): - raise RuntimeError( - "Numba integration for cppyy is experimental and only tested on Linux x86_64. " - f"You are running on '{platform.machine()}', so behavior may be incorrect or fail. " - "See https://cppyy.readthedocs.io/en/latest/numba.html#numba-support" - ) - nbjit = nb.jit(nopython=True, inline="always")(func) - # In this case, the user has to explictly provide the return type, cannot be inferred - if return_type is None: - raise RuntimeError( - "Failed to infer the return type for the provided function. " - "Please specify the signature explicitly in the decorator, e.g.: " - "@ROOT.NumbaDeclare(['double'], 'double')" - ) - except: # noqa E722 - raise Exception("Failed to jit Python callable {} with numba.jit".format(func)) - func.numba_func = nbjit - if return_type is None: - type_map = { - nb.types.boolean: "bool", - nb.types.uint8: "unsigned int", - nb.types.uint16: "unsigned int", - nb.types.uint32: "unsigned int", - nb.types.uint64: "unsigned long", - nb.types.char: "int", - nb.types.int8: "int", - nb.types.int16: "int", - nb.types.int32: "int", - nb.types.int64: "long", - nb.types.float32: "float", - nb.types.float64: "double", - } - - if nb_return_type in type_map: - return_type = type_map[nb_return_type] - elif "array" in nb.typeof(nb_return_type).name: - return_type = "RVec<" + type_map[nb_return_type.dtype] + ">" - # Create Python wrapper with C++ friendly signature - - # Define signature - pywrapper_signature = [ - "ptr_{0}, size_{0}".format(i) if is_container_type(t) else "x_{}".format(i) - for i, t in enumerate(input_types) - ] - if is_container_type(return_type): - # If we return a container, we return via pointer the pointer of the allocated data, - # the size in elements. In addition, we provide the size of the datatype in bytes. - pywrapper_signature += ["ptrptr_r, ptrsize_r"] - - # Define arguments for jit function - pywrapper_args_def = [ - "x_{0} = nb.carray(ptr_{0}, (size_{0},))".format(i) if is_container_type(t) else "x_{}".format(i) - for i, t in enumerate(input_types) - ] - pywrapper_args = ["x_{}".format(i) for i in range(len(input_types))] - - # Define return operation - if is_container_type(return_type): - innert = get_inner_type(return_type) - dtypesize = 1 if innert == "bool" else int(map_cpp_type(innert).bitwidth / 8) - pywrapper_return = "\n ".join( - [ - "# Because we cannot manipulate the memory management of the numpy array we copy the data", - "ptr = malloc(r.size * {})".format(dtypesize), - "cp = nb.carray(ptr, r.size, dtype_r)", - "cp[:] = r[:]", - "# Return size of the array and the pointer to the copied data", - "ptrsize_r[0] = r.size", - "ptrptr_r[0] = cp.ctypes.data", - ] - ) - else: - pywrapper_return = "return r" - - # bind the arguments that are passed as void* pointers to the correct type - pywrapper_bind_lines = "\n ".join( - f'x_{i} = ROOT._cppyy.bind_object(x_{i}, "{t}")' - for i, t in enumerate(input_types) - if not is_container_type(t) and map_cpp_type(t, as_ctypes=True) == ctypes.c_void_p - ) - # Only import ROOT if we have objects to bind. - # Importing ROOT inside a Numba cfunc call will fail, so we do it only if we know - # we will actually go through ctypes.CFUNTYPE - if pywrapper_bind_lines: - pywrapper_bind_lines = "import ROOT\n " + pywrapper_bind_lines - - # Build wrapper code - pywrappercode = '''\ -def pywrapper({SIGNATURE}): - """ - Wrapper function for the jitted Python callable with special treatment of arrays - """ - # If a container is given, define numba carray wrapper for the input types - {ARGS_DEF} - # If an object was passed as a void*, bind it to the correct type - {BIND_LINES} - # Call the jitted Python function - r = nbjit({ARGS}) - # Return the result - {RETURN} - '''.format( - SIGNATURE=", ".join(pywrapper_signature), - ARGS_DEF="\n ".join(pywrapper_args_def), - BIND_LINES=pywrapper_bind_lines, - ARGS=", ".join(pywrapper_args), - RETURN=pywrapper_return, - ) - - glob = dict(globals()) # Make a shallow copy of the dictionary so we don't pollute the global scope - glob["nb"] = nb - glob["nbjit"] = nbjit - - ffi = cffi.FFI() - ffi.cdef("void* malloc(long size);") - C = ffi.dlopen(None) - glob["malloc"] = C.malloc - - if is_container_type(return_type): - glob["dtype_r"] = map_cpp_type(get_inner_type(return_type)) - - # Execute the pywrapper code and generate the wrapper function - # which calls the jitted C function - # Python 3.13 changes the semantics of the `locals()` builtin function - # such that in optimized scopes (e.g. at function scope as it is - # happening right now) the dictionary is not updated when changed. Bind - # the `locals()` dictionary to a temporary object. This way, the call - # to `exec()` will actually change the dictionary and the pywrapper - # function will be found. Note that this change is backwards-compatible. - local_objects = locals() - exec(pywrappercode, glob, local_objects) - - if "pywrapper" not in local_objects: - raise Exception("Failed to create Python wrapper function:\n{}".format(pywrappercode)) - - # Jit the Python wrapper code - c_return_type, c_input_types = get_c_signature(input_types, return_type) - try: - nbcfunc = nb.cfunc(c_return_type(*c_input_types), nopython=True)(local_objects["pywrapper"]) - except: # noqa E722 - try: - # Fallback to using ctypes.CFUNCTYPE if numba.cfunc fails - c_return_type, c_input_types = get_c_signature(input_types, return_type, as_ctypes=True) - cfunc_type = ctypes.CFUNCTYPE(c_return_type, *c_input_types) - nbcfunc = cfunc_type(local_objects["pywrapper"]) - except: # noqa E722 - raise Exception("Failed to jit Python wrapper with numba.cfunc") - func.__py_wrapper__ = pywrappercode - func.__numba_cfunc__ = nbcfunc - - # Get address of jitted wrapper function - address = nbcfunc.address if hasattr(nbcfunc, "address") else ctypes.cast(nbcfunc, ctypes.c_void_p).value - - # Infer name of the C++ wrapper function - if not name: - name = func.__name__ - - # Build C++ wrapper for jitting with cling - - # Define: - # - Input signature - # - Function pointer types - input_types_ref = [] - func_ptr_input_types = [] - for t in input_types: - container_match_group = "|".join(entry["match_pattern"] for entry in CONTAINER_TYPES.values()) - m = re.match(rf"\s*(const\s+)?({container_match_group})", t) - if m: - const_mod = "" if m.group(1) is None else "const " - container_t = m.group(2) - # add namespaces if missing - container_t = normalize_container_type(container_t) - - add_container_input_type_ref(input_types_ref, const_mod, container_t) - add_container_func_ptr_input_type(func_ptr_input_types, const_mod, container_t) - else: - input_types_ref.append(t) - func_ptr_input_types.append(t) - - input_signature = ", ".join("{} x_{}".format(t, i) for i, t in enumerate(input_types_ref)) - - if is_container_type(return_type): - return_type = normalize_container_type(return_type) - # See C++ wrapper code for the reason using these types - innert = get_inner_type(return_type) - func_ptr_input_types += ["{}**, long*".format("char" if innert == "bool" else innert)] - func_ptr_type = "{RETURN_TYPE}(*)({INPUT_TYPES})".format( - RETURN_TYPE="void*" if is_container_type(return_type) else return_type, - INPUT_TYPES=", ".join(func_ptr_input_types), - ) - - # Define function call - vecbool_conversion = [] - func_args = [] - for i, t in enumerate(input_types): - if is_container_type(t): - func_args += ["x_{0}.data(), x_{0}.size()".format(i)] - if get_inner_type(t) == "bool": - # Copy the container to a container to match the numpy memory layout - func_args[-1] = func_args[-1].replace("x_", "xb_") - vecbool_conversion += [f"{get_container_cpp_names(t)[0]} xb_{i} = x_{i};"] - else: - func_args += ["x_{}".format(i)] - if is_container_type(return_type): - # See C++ wrapper code for the reason using these arguments - func_args += ["&ptr, &size"] - - # Define return operation - if is_container_type(return_type): - innert = get_inner_type(return_type) - container_cpp = get_container_cpp_names(return_type)[0] - inner_with_dims = get_inner_type(return_type, with_dims=True) - - if innert == "bool": - innert = "char" - inner_with_dims = "char" + inner_with_dims[4:] - - # Default case: create container from pointer and size (e.g. std::vector, RVec) - copy_lines = [f"{container_cpp}<{innert}> x_r(ptr, ptr + size);"] - - # Special case: if inner type includes dimensions (e.g. std::array) - # We construct the array then copy the data manually - if innert != inner_with_dims: - innert, dims = inner_with_dims.split(",") - copy_lines = [f"{container_cpp}<{innert}, {dims}> x_r;std::copy(ptr, ptr + size, x_r.begin());"] - - return_op = "\n ".join( - [ - "// Because a container type cannot take the ownership of external data, we have to copy the returned array", - "long size; // Size of the returned array", - f"{innert}* ptr; // Pointer to the data of the returned array", - f"funcptr({', '.join(func_args)});", - # TODO: Remove this copy as soon as the container type can adopt the ownership - *copy_lines, - "free(ptr);", - # If we return a container_type, we rely here on the automatic conversion of container_type to container_type - "return x_r;", - ] - ) - else: - return_op = "return funcptr({});".format(", ".join(func_args)) - - # Build wrapper code - cppwrappercode = """\ -namespace Numba {{ -/* - * C++ wrapper function around the jitted Python wrapping which calls the jitted Python callable - */ -{RETURN_TYPE} {FUNC_NAME}({INPUT_SIGNATURE}) {{ - // Create a function pointer from the jitted Python wrapper - const auto funcptr = reinterpret_cast<{FUNC_PTR_TYPE}>({FUNC_PTR}); - // Perform conversion of container_type - {VECBOOL_CONVERSION} - // Return the result - {RETURN_OP} -}} -}}""".format( - RETURN_TYPE=return_type, - FUNC_NAME=name, - INPUT_SIGNATURE=input_signature, - FUNC_PTR=address, - FUNC_PTR_TYPE=func_ptr_type, - VECBOOL_CONVERSION="\n ".join(vecbool_conversion), - RETURN_OP=return_op, - ) - - # Jit wrapper C++ code - err = ROOT.gInterpreter.Declare(cppwrappercode) - if not err: - raise Exception("Failed to jit C++ wrapper code with cling:\n{}".format(cppwrappercode)) - func.__cpp_wrapper__ = cppwrappercode - - return func - - return inner diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py index 657cb7201690a..9dcf4cd254836 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/__init__.py @@ -41,7 +41,7 @@ def pypow(x, y): from .ast_backend import CPP_KEYWORDS, AstTranspiler from .errors import PyDeclareError -__all__ = ["Declare", "PyDeclareError", "declare"] +__all__ = ["Declare", "DeclareInNumbaNamespace", "PyDeclareError", "declare"] DEFAULT_NAMESPACE = "Py" @@ -202,18 +202,24 @@ def inner(func): return inner -def _numba_declare_dispatch(input_types, return_type=None, name=None, **kwargs): - """``ROOT.Numba.Declare``, routed to the implementation selected by - ``$ROOT_NUMBA_DECLARE_BACKEND``: ``numba`` (the default, the original - implementation) or ``ast``. The generated function always lands in the - ``Numba`` C++ namespace, so existing code that says ``"Numba::myfunc(x)"`` - keeps working with either implementation. +def DeclareInNumbaNamespace(input_types=None, return_type=None, name=None): + """Decorator making a Python callable available in C++, by translation. + + This is ``ROOT.Numba.Declare``. The decorator used to compile the callable + with numba, hence the name. It no longer does, but the spelling and the + ``Numba`` C++ namespace are kept so that existing code saying + ``"Numba::myfunc(x)"`` keeps working unchanged. ``ROOT.Py.Declare`` is the + same thing under a name that is not a historical accident. + + input_types + List of C++ type names, one per argument of the callable. + return_type + The C++ return type. If omitted, it is deduced -- either by the + transpiler, or by the C++ compiler through an ``auto`` return type. + name + The name of the generated C++ function; defaults to the Python name. + + The generated C++ source is available on the decorated callable as + ``__cpp_wrapper__``. """ - backend = os.environ.get("ROOT_NUMBA_DECLARE_BACKEND", "numba").strip().lower() - if backend in ("", "numba"): - from .._numbadeclare import _NumbaDeclareDecorator - - return _NumbaDeclareDecorator(input_types, return_type, name) - if backend != "ast": - raise PyDeclareError("ROOT_NUMBA_DECLARE_BACKEND is '{}'; expected 'numba' or 'ast'".format(backend)) - return Declare(input_types, return_type, name, namespace="Numba", **kwargs) + return Declare(input_types, return_type, name, namespace="Numba") diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py index 21351044a3de3..ab6b98ed92bc2 100755 --- a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py @@ -11,12 +11,12 @@ import re import typing -from .._pydeclare import _numba_declare_dispatch +from .._pydeclare import DeclareInNumbaNamespace class FunctionJitter: """ - This class allows to jit a python callable with Numba, being able to infer the signature of the function from the types of the RDF columns. + This class allows to translate a python callable to C++, being able to infer the signature of the function from the types of the RDF columns. It takes a python callable, an RDF and optionally a column list and dictionary containing any other arguments for the callable. It determines the function signature and Declares the function and returns the corresponding function call. @@ -94,7 +94,20 @@ def find_type(self, x): return t else: return "str" - #! Numba Declare does not support "string" type. Check _numbadeclare.Thus, Cannot pass string constants to the filter/Defines.. + #! The translation does not support the "string" type, so string constants + #! cannot be passed to the Filters/Defines. + elif type(x) is int: + # A Python int has no width; pick a C++ type that can hold this one, + # rather than truncating anything above 2**31 to a negative int. + if -(2**31) <= x < 2**31: + return "int" + if 2**31 <= x < 2**32: + return "unsigned int" + if -(2**63) <= x < 2**63: + return "long long" + if 2**63 <= x < 2**64: + return "unsigned long long" + raise TypeError("The integer {} does not fit in any C++ integer type.".format(x)) elif type(x) in FUNDAMENTAL_PYTHON_TYPES: return FUNDAMENTAL_PYTHON_TYPES[type(x)] elif isinstance(x, np.ndarray): @@ -208,7 +221,7 @@ def get_function_params_args_call(self, func, cols_list, extra_args): def jit_function(self, func, cols_list, extra_args): """ - Jits the provided function using ROOT's NumbaDeclare. + Declares the provided function as C++ using ROOT's Numba.Declare. Also checks if the function was jitted earlier in which case it won't jit again but if signature does not match. It raises an error. Arguments: @@ -226,7 +239,7 @@ def jit_function(self, func, cols_list, extra_args): self.get_function_params_args_call(func, cols_list, extra_args) if not hasattr(self.func, "__cpp_wrapper__"): # not already declared with ROOT.Numba.Declare - _numba_declare_dispatch(self.func_sign, self.return_type)(self.func) + DeclareInNumbaNamespace(self.func_sign, self.return_type)(self.func) FunctionJitter.function_cache[self.func.__name__] = (self.func_call, self.func_sign) return self.func_call diff --git a/bindings/pyroot/pythonizations/test/CMakeLists.txt b/bindings/pyroot/pythonizations/test/CMakeLists.txt index c803145e4f8a5..e8f9fee239af6 100644 --- a/bindings/pyroot/pythonizations/test/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/test/CMakeLists.txt @@ -115,16 +115,10 @@ if(roofit) endif() if (dataframe) - # Test wrapping Python callables for use in C++ using numba - ROOT_ADD_PYUNITTEST(pyroot_numbadeclare numbadeclare.py PYTHON_DEPS numba) - # The same suite, run against both implementations of Numba.Declare: - # the numba one and the transpiler, which needs no numba. - ROOT_ADD_PYUNITTEST(pyroot_pydeclare_numba pydeclare.py PYTHON_DEPS numba - ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=numba) - ROOT_ADD_PYUNITTEST(pyroot_pydeclare_ast pydeclare.py - ENVIRONMENT ROOT_NUMBA_DECLARE_BACKEND=ast) - ROOT_ADD_PYUNITTEST(pyroot_rdf_filter_pyz rdf_filter_pyz.py PYTHON_DEPS numba) - ROOT_ADD_PYUNITTEST(pyroot_rdf_define_pyz rdf_define_pyz.py PYTHON_DEPS numba) + # Test declaring C++ functions from Python callables + ROOT_ADD_PYUNITTEST(pyroot_pydeclare pydeclare.py) + ROOT_ADD_PYUNITTEST(pyroot_rdf_filter_pyz rdf_filter_pyz.py) + ROOT_ADD_PYUNITTEST(pyroot_rdf_define_pyz rdf_define_pyz.py) # std::string_view in CPyCppyy ROOT_ADD_PYUNITTEST(pyroot_string_view string_view.py) endif() diff --git a/bindings/pyroot/pythonizations/test/numbadeclare.py b/bindings/pyroot/pythonizations/test/numbadeclare.py deleted file mode 100644 index c8bc5684356e3..0000000000000 --- a/bindings/pyroot/pythonizations/test/numbadeclare.py +++ /dev/null @@ -1,751 +0,0 @@ -import gc -import platform -import sys -import unittest - -import numba as nb -import numpy as np -import ROOT - -default_test_inputs = [-1.0, 0.0, 100.0] - - -class NumbaDeclareSimple(unittest.TestCase): - """ - Test decorator to create C++ wrapper for Python callables using numba with fundamental types - """ - - test_inputs = default_test_inputs - - # Test refcounts - def test_refcount_decorator(self): - """ - Test refcount of decorator - - In case of Python < 3.14, we expect a refcount of 2, because the call - to sys.getrefcount can create an additional reference itself. Starting - from Python 3.14, we expect a refcount of 1 because there were changes - to the interpreter to avoid some unnecessary ref counts. See also: - https://docs.python.org/3.14/whatsnew/3.14.html#whatsnew314-refcount - """ - x = ROOT.Numba.Declare(["float"], "float") - gc.collect() - extra_ref_count = int(sys.version_info < (3, 14)) - self.assertEqual(sys.getrefcount(x), 1 + extra_ref_count) - - def test_refcount_pycallable(self): - """ - Test refcount of decorated callable - """ - - def f1(x): - return x - - def f2(x): - return x - - fn0 = ROOT.Numba.Declare(["float"], "float")(f1) # noqa: F841 - ref = nb.cfunc("float32(float32)", nopython=True)(f2) # noqa: F841 - gc.collect() - self.assertEqual(sys.getrefcount(f1), sys.getrefcount(f2) + 2) - - # Test optional name - def test_optional_name(self): - """ - Test optional name of wrapper function - """ - optname = "optname2" - - @ROOT.Numba.Declare(["float"], "float", name=optname) - def f(x): - return x - - self.assertTrue(hasattr(ROOT.Numba, optname)) - - # Test attributes - def test_additional_attributes(self): - """ - Test additional attributes - """ - - @ROOT.Numba.Declare(["float"], "float") - def fn1(x): - return x - - gc.collect() - - self.assertTrue(hasattr(fn1, "__cpp_wrapper__")) - self.assertTrue(isinstance(fn1.__cpp_wrapper__, str)) - self.assertLessEqual(sys.getrefcount(fn1.__cpp_wrapper__), 3) - - self.assertTrue(hasattr(fn1, "__py_wrapper__")) - self.assertTrue(isinstance(fn1.__py_wrapper__, str)) - self.assertEqual(sys.getrefcount(fn1.__py_wrapper__), 2) - - self.assertTrue(hasattr(fn1, "numba_func")) - self.assertEqual(sys.getrefcount(fn1.numba_func), 3) - - # Test cling integration - def test_cling(self): - """ - Test function call in cling - """ - - @ROOT.Numba.Declare(["float"], "float") - def fn12(x): - return 2.0 * x - - ROOT.gInterpreter.ProcessLine("y12 = Numba::fn12(42.0);") - self.assertEqual(fn12(42.0), ROOT.y12) - - # Test RDataFrame integration - def test_rdataframe(self): - """ - Test function call as part of RDataFrame - """ - - @ROOT.Numba.Declare(["unsigned int"], "float") - def fn13(x): - return 2.0 * x - - df = ROOT.RDataFrame(4).Define("x", "rdfentry_").Define("y", "Numba::fn13(x)") - mean_x = df.Mean("x") - mean_y = df.Mean("y") - self.assertEqual(mean_x.GetValue(), 1.5) - self.assertEqual(mean_y.GetValue(), 3.0) - - def test_rdataframe_temporary(self): - """ - Test passing a temporary from an RDataFrame operation - """ - - @ROOT.Numba.Declare(["const RVecF"], "RVecF") - def pass_temporary(v): - return v * np.array([2.0]).astype(np.float32) - - df = ROOT.RDataFrame(1).Define("v", "ROOT::RVecF{0.f,2.f}").Define("v2", "Numba::pass_temporary(v[v > 0])") - - rvecf = df.Take["RVecF"]("v2").GetValue()[0] - - self.assertTrue(np.array_equal(rvecf, np.array([4.0]))) - - def test_rdataframe_std_vector(self): - """ - Test function call as part of RDataFrame - """ - - @ROOT.Numba.Declare(["std::vector"], "std::vector") - def square_vec(x): - return x * x - - df = ROOT.RDataFrame(4).Define("x", "std::vector{1, 2, 3}").Define("x_sq", "Numba::square_vec(x)") - df.Display().Print() - self.assertEqual(df.Sum("x").GetValue(), 24) - self.assertEqual(df.Sum("x_sq").GetValue(), 56) - - def test_rdataframe_std_array(self): - """ - Test function call as part of RDataFrame with std::array - """ - - @ROOT.Numba.Declare(["std::array"], "std::array") - def square_array(x): - return x * x - - df = ROOT.RDataFrame(4).Define("x", "std::array{1, 2, 3}").Define("x_sq", "Numba::square_array(x)") - df.Display().Print() - self.assertEqual(df.Sum("x").GetValue(), 24) - self.assertEqual(df.Sum("x_sq").GetValue(), 56) - - @unittest.skipIf( - not (platform.system() == "Linux" and platform.machine() == "x86_64"), - "Numba integration is experimental and only tested on Linux x86_64", - ) - def test_rdataframe_LorentzVector(self): - """ - Test function call as part of RDataFrame with ROOT::Math::LorentzVector - """ - - @ROOT.Numba.Declare(["ROOT::Math::PtEtaPhiMVector"], "float") - def get_m(v): - return v.M() - - df = ROOT.RDataFrame(4).Define("v", "ROOT::Math::PtEtaPhiMVector(1, 2, 3, 4)").Define("v_m", "Numba::get_m(v)") - - self.assertEqual(df.Sum("v_m").GetValue(), 16.0) - - # Test wrappings - def test_wrapper_in_void(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare([], "float") - def fn2n(): - return float(42) - - x1 = fn2n() - x2 = ROOT.Numba.fn2n() - self.assertEqual(x1, x2) - self.assertEqual(type(x1), type(x2)) - - def test_wrapper_out_f(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["float"], "float") - def fn2(x): - return float(x) - - for v in self.test_inputs: - x1 = fn2(v) - x2 = ROOT.Numba.fn2(v) - self.assertEqual(x1, x2) - self.assertEqual(type(x1), type(x2)) - - def test_wrapper_out_d(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["float"], "double") - def fn2d(x): - return float(x) - - for v in self.test_inputs: - x1 = fn2d(v) - x2 = ROOT.Numba.fn2d(v) - self.assertEqual(x1, x2) - # NOTE: There is no double in Python because everything is a double. - self.assertEqual(type(x1), type(x2)) - - def test_wrapper_out_i(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["float"], "int") - def fn3(x): - return int(x) - - for v in self.test_inputs: - x1 = fn3(v) - x2 = ROOT.Numba.fn3(v) - self.assertEqual(x1, x2) - self.assertEqual(type(x1), type(x2)) - - def test_wrapper_out_l(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["float"], "long") - def fn4(x): - return x - - for v in self.test_inputs: - x1 = fn4(v) - x2 = ROOT.Numba.fn4(v) - self.assertEqual(x1, x2) - self.assertEqual(int, type(x2)) - - def test_wrapper_out_u(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["float"], "unsigned int") - def fn5(x): - return abs(x) - - for v in self.test_inputs: - x1 = fn5(v) - x2 = ROOT.Numba.fn5(v) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), int) - - def test_wrapper_out_k(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["float"], "unsigned long") - def fn6(x): - return abs(x) - - for v in self.test_inputs: - x1 = fn6(v) - x2 = ROOT.Numba.fn6(v) - self.assertEqual(x1, x2) - self.assertEqual(int, type(x2)) - - def test_wrapper_out_b(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["float"], "bool") - def fn6b(x): - return x > 0 - - for v in self.test_inputs: - x1 = fn6b(v) - x2 = ROOT.Numba.fn6b(v) - self.assertEqual(x1, x2) - self.assertEqual(type(x1), type(x2)) - - def test_wrapper_in_b(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["bool"], "bool") - def fn6b2(x): - return not x - - for v in [True, False]: - x1 = fn6b2(v) - x2 = ROOT.Numba.fn6b2(v) - self.assertEqual(x1, x2) - self.assertEqual(type(x1), type(x2)) - - def test_wrapper_in_i(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["int"], "float") - def fn7i(x): - return 2.0 * x - - for v in [-1, 0, 1, 999]: - x1 = fn7i(v) - x2 = ROOT.Numba.fn7i(v) - self.assertEqual(x1, x2) - - def test_wrapper_in_l(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["long"], "float") - def fn7l(x): - return 2.0 * x - - for v in [-1, 0, 1, 999]: - x1 = fn7l(v) - x2 = ROOT.Numba.fn7l(v) - self.assertEqual(x1, x2) - - def test_wrapper_in_ui(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["unsigned int"], "float") - def fn7ui(x): - return 2.0 * x - - for v in [0, 1, 999]: - x1 = fn7ui(v) - x2 = ROOT.Numba.fn7ui(v) - self.assertEqual(x1, x2) - - def test_wrapper_in_ul(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["unsigned long"], "float") - def fn7ul(x): - return 2.0 * x - - for v in [0, 1, 999]: - x1 = fn7ul(v) - x2 = ROOT.Numba.fn7ul(v) - self.assertEqual(x1, x2) - - -class NumbaDeclareArray(unittest.TestCase): - """ - Test decorator to create C++ wrapper for Python callables using numba with RVecs - """ - - test_inputs = [default_test_inputs] - - # The global module index does not have RVec entities preloaded and - # gInterpreter.Declare is not allowed to load libROOTVecOps for RVec. - # Preload the library now. - ROOT.gSystem.Load("libROOTVecOps") - - def test_wrapper_in_vecf(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "float") - def g1(x): - return x.sum() - - for v in self.test_inputs: - x1 = g1(np.array(v, dtype=np.float32)) - x2 = ROOT.Numba.g1(ROOT.VecOps.RVec("float")(v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), float) - - def test_wrapper_in_vecf_vecd(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec", "RVec"], "float") - def g1_2vec(x, y): - return x.sum() + y.sum() - - for v in self.test_inputs: - x1 = g1_2vec(np.array(v, dtype=np.float32), np.array(v, dtype=np.float64)) - x2 = ROOT.Numba.g1_2vec(ROOT.VecOps.RVec("float")(v), ROOT.VecOps.RVec("double")(v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), float) - - def test_wrapper_in_vecd(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "float") - def g1d(x): - return x.sum() - - for v in self.test_inputs: - x1 = g1d(np.array(v, dtype=np.float64)) - x2 = ROOT.Numba.g1d(ROOT.VecOps.RVec("double")(v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), float) - - def test_wrapper_in_veci(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "int") - def g1i(x): - return x.sum() - - for v in self.test_inputs: - x1 = g1i(np.array(v, dtype=np.int32)) - x2 = ROOT.Numba.g1i(ROOT.VecOps.RVec("int")(int(x) for x in v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), int) - - def test_wrapper_in_vecl(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "int") - def g1l(x): - return x.sum() - - for v in self.test_inputs: - x1 = g1l(np.array(v, dtype=np.int64)) - x2 = ROOT.Numba.g1l(ROOT.VecOps.RVec("long")(int(x) for x in v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), int) - - def test_wrapper_in_vecui(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "int") - def g1ui(x): - return x.sum() - - for v in [[0, 1, 999]]: - x1 = g1ui(np.array(v, dtype=np.uint32)) - x2 = ROOT.Numba.g1ui(ROOT.VecOps.RVec("unsigned int")(v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), int) - - def test_wrapper_in_vecul(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "int") - def g1ul(x): - return x.sum() - - for v in [[0, 1, 999]]: - x1 = g1ul(np.array(v, dtype=np.uint64)) - x2 = ROOT.Numba.g1ul(ROOT.VecOps.RVec("unsigned long")(v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), int) - - def test_wrapper_in_vecb(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "int") - def g1b(x): - return x.sum() - - for v in [[True, False, True]]: - x1 = g1b(np.array(v, dtype=np.float32)) - x2 = ROOT.Numba.g1b(ROOT.VecOps.RVec("bool")(v)) - self.assertEqual(x1, x2) - self.assertEqual(type(x2), int) - - def test_wrapper_out_vecf(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "RVec") - def g2f(x): - return x[::-1] - - for v in [[0, 1, 999]]: - x1 = g2f(np.array(v, dtype=np.float32)) - x2 = ROOT.Numba.g2f(ROOT.VecOps.RVec("float")(v)) - self.assertTrue((x1 == x2).all()) - - def test_wrapper_out_vecd(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "RVec") - def g2d(x): - return x[::-1] - - for v in [[0, 1, 999]]: - x1 = g2d(np.array(v, dtype=np.float64)) - x2 = ROOT.Numba.g2d(ROOT.VecOps.RVec("double")(v)) - self.assertTrue((x1 == x2).all()) - - def test_wrapper_out_veci(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "RVec") - def g2i(x): - return x[::-1] - - for v in [[0, 1, 999]]: - x1 = g2i(np.array(v, dtype=np.int32)) - x2 = ROOT.Numba.g2i(ROOT.VecOps.RVec("int")(v)) - self.assertTrue((x1 == x2).all()) - - def test_wrapper_out_vecl(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "RVec") - def g2l(x): - return x[::-1] - - for v in [[0, 1, 999]]: - x1 = g2l(np.array(v, dtype=np.int64)) - x2 = ROOT.Numba.g2l(ROOT.VecOps.RVec("long")(v)) - self.assertTrue((x1 == x2).all()) - - def test_wrapper_out_vecul(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "RVec") - def g2ul(x): - return x[::-1] - - for v in [[0, 1, 999]]: - x1 = g2ul(np.array(v, dtype=np.uint64)) - x2 = ROOT.Numba.g2ul(ROOT.VecOps.RVec("unsigned long")(v)) - self.assertTrue((x1 == x2).all()) - - def test_wrapper_out_vecui(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "RVec") - def g2ui(x): - return x[::-1] - - for v in [[0, 1, 999]]: - x1 = g2ui(np.array(v, dtype=np.uint32)) - x2 = ROOT.Numba.g2ui(ROOT.VecOps.RVec("unsigned int")(v)) - self.assertTrue((x1 == x2).all()) - - def test_wrapper_out_vecb(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec"], "RVec") - def g2b(x): - return x[::-1] - - for v in [[True, False]]: - x1 = g2b(np.array(v, dtype=bool)) - x2 = ROOT.Numba.g2b(ROOT.VecOps.RVec("bool")(v)) - self.assertEqual(x1[0], bool(x2[0])) - self.assertEqual(x1[1], bool(x2[1])) - - def test_wrapper_in_vecfb_out_vecf(self): - """ - Test wrapper with different input/output configurations - """ - - @ROOT.Numba.Declare(["RVec", "RVec"], "RVec") - def g2fb(x, y): - return (x > 1) | y - - for vf, vb in [[[1.0, 2.0], [True, False]]]: - x1 = g2fb(np.array(vf, dtype=np.float32), np.array(vb, dtype=bool)) - x2 = ROOT.Numba.g2fb(ROOT.VecOps.RVec("float")(vf), ROOT.VecOps.RVec("bool")(vb)) - self.assertEqual(x1[0], bool(x2[0])) - self.assertEqual(x1[1], bool(x2[1])) - - def test_const_modifier(self): - """ - Test const modifier in input argument type - """ - - @ROOT.Numba.Declare(["const ROOT::VecOps::RVec"], "RVecF") - def const_mod(v): - return v * np.array([1.0, 2.0]).astype(np.float32) - - rvecf = ROOT.Numba.const_mod(ROOT.RVecF([1.0, 2.0])) - - self.assertTrue(np.array_equal(rvecf, np.array([1.0, 4.0]))) - - def test_reference(self): - """ - Test passing a reference as input argument - """ - - @ROOT.Numba.Declare(["RVec&"], "RVecF") - def pass_reference(v): - return v * np.array([1.0, 2.0]).astype(np.float32) - - rvecf = ROOT.Numba.pass_reference(ROOT.RVecF([1.0, 2.0])) - - self.assertTrue(np.array_equal(rvecf, np.array([1.0, 4.0]))) - - -class NumbaDeclareInferred(unittest.TestCase): - """ - Test decorator created with a reconstructed list of arguments using RDF column types, - and a return type inferred from the numba jitted function. - """ - - def test_fund_types(self): - """ - Test fundamental types - """ - df = ROOT.RDataFrame(4).Define("x", "rdfentry_") - - with self.subTest("function"): - - @ROOT.Numba.Declare(["unsigned long"]) - def is_even(x): - return x % 2 == 0 - - df = df.Define("is_even_x_1", "Numba::is_even(x)") - results = df.Take["bool"]("is_even_x_1").GetValue()[0] - self.assertEqual(results, True) - - with self.subTest("lambda"): - ROOT.Numba.Declare(["unsigned long"], name="is_even_lambda")(lambda x: x % 2 == 0) - df = df.Define("is_even_x_2", "Numba::is_even_lambda(x)") - results = df.Take["bool"]("is_even_x_2").GetValue()[0] - self.assertEqual(results, True) - - def test_rvec(self): - """ - Test RVec - """ - df = ROOT.RDataFrame(4).Define("x", "ROOT::VecOps::RVec({1, 2, 3})") - - with self.subTest("function"): - - @ROOT.Numba.Declare(["RVec"]) - def square_rvec(v): - return v * v - - df = df.Define("square_rvec_1", "Numba::square_rvec(x)") - results = df.Take["RVec"]("square_rvec_1").GetValue()[0] - self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) - - with self.subTest("lambda"): - ROOT.Numba.Declare(["RVec"], name="square_rvec_lambda")(lambda v: v * v) - df = df.Define("square_rvec_2", "Numba::square_rvec_lambda(x)") - results = df.Take["RVec"]("square_rvec_2").GetValue()[0] - self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) - - def test_std_vec(self): - """ - Test std::vector - """ - df = ROOT.RDataFrame(4).Define("x", "std::vector({1, 2, 3})") - - with self.subTest("function"): - - @ROOT.Numba.Declare(["std::vector"]) - def square_std_vec(v): - return v * v - - df = df.Define("square_std_vec_1", "Numba::square_std_vec(x)") - results = df.Take["RVec"]("square_std_vec_1").GetValue()[0] - self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) - - with self.subTest("lambda"): - ROOT.Numba.Declare(["std::vector"], name="square_std_vec_lambda")(lambda v: v * v) - df = df.Define("square_std_vec_2", "Numba::square_std_vec_lambda(x)") - results = df.Take["RVec"]("square_std_vec_2").GetValue()[0] - self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) - - def test_std_array(self): - """ - Test std::array - """ - df = ROOT.RDataFrame(4).Define("x", "std::array({1, 2, 3})") - - with self.subTest("function"): - - @ROOT.Numba.Declare(["std::array"]) - def square_std_arr(v): - return v * v - - df = df.Define("square_std_arr_1", "Numba::square_std_arr(x)") - results = df.Take["RVec"]("square_std_arr_1").GetValue()[0] - self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) - - with self.subTest("lambda"): - ROOT.Numba.Declare(["std::array"], name="square_std_arr_lambda")(lambda v: v * v) - df = df.Define("square_std_arr_2", "Numba::square_std_arr_lambda(x)") - results = df.Take["RVec"]("square_std_arr_2").GetValue()[0] - self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) - - def test_missing_signature_raises(self): - """ - Ensure an Exception is raised when return type cannot be inferred - and no explicit signature is provided in the decorator. - """ - - with self.assertRaises(Exception): - - @ROOT.Numba.Declare(["ROOT::Math::PtEtaPhiMVector"]) - def f(x): - return x.M() - - -if __name__ == "__main__": - unittest.main() diff --git a/bindings/pyroot/pythonizations/test/pydeclare.py b/bindings/pyroot/pythonizations/test/pydeclare.py index 20c7fdc051ba3..be4f4f0917732 100644 --- a/bindings/pyroot/pythonizations/test/pydeclare.py +++ b/bindings/pyroot/pythonizations/test/pydeclare.py @@ -1,42 +1,35 @@ -"""Backend-parameterised version of numbadeclare.py. - -The same suite is run against both implementations of ROOT.Numba.Declare: the -original numba one, and the transpiler that translates the Python callable to -C++ source instead. Which one is exercised is selected by -$ROOT_NUMBA_DECLARE_BACKEND, read by the decorator itself, so apart from the -handful of assertions that are about numba's implementation rather than about -the feature, this file is the original test suite unchanged. +"""Tests for declaring C++ functions from Python callables. + +The bulk of this file is the suite that used to test the numba based +implementation of ROOT.Numba.Declare, unchanged: the decorator now translates +the callable to C++ source instead of compiling it with numba, and the point of +keeping the assertions as they were is that this is not supposed to be visible +from the outside. + +PyDeclareNamespace at the end covers what is new -- the ROOT.Py.Declare +spelling, the constructs the translation supports that numba never did, and the +requirement that anything outside the supported subset is refused loudly. """ import gc +import importlib.util import os -import platform +import shutil import sys +import tempfile +import textwrap import unittest import numpy as np import ROOT - -BACKEND = os.environ.get("ROOT_NUMBA_DECLARE_BACKEND", "numba").strip().lower() -IS_NUMBA = BACKEND in ("", "numba") - -if IS_NUMBA: - import numba as nb - -# The transpiler backends translate the callable to C++ source, so they are not -# restricted to the platforms the numba/cppyy bridge has been tested on. -numba_only = unittest.skipUnless(IS_NUMBA, "tests numba's implementation specifically") -needs_cppyy_numba_ext = unittest.skipIf( - IS_NUMBA and not (platform.system() == "Linux" and platform.machine() == "x86_64"), - "Numba integration is experimental and only tested on Linux x86_64", -) +from ROOT._pydeclare import PyDeclareError default_test_inputs = [-1.0, 0.0, 100.0] class NumbaDeclareSimple(unittest.TestCase): """ - Test decorator to create C++ wrapper for Python callables using numba with fundamental types + Test decorator to create C++ wrapper for Python callables with fundamental types """ test_inputs = default_test_inputs @@ -57,23 +50,6 @@ def test_refcount_decorator(self): extra_ref_count = int(sys.version_info < (3, 14)) self.assertEqual(sys.getrefcount(x), 1 + extra_ref_count) - @numba_only - def test_refcount_pycallable(self): - """ - Test refcount of decorated callable - """ - - def f1(x): - return x - - def f2(x): - return x - - fn0 = ROOT.Numba.Declare(["float"], "float")(f1) # noqa: F841 - ref = nb.cfunc("float32(float32)", nopython=True)(f2) # noqa: F841 - gc.collect() - self.assertEqual(sys.getrefcount(f1), sys.getrefcount(f2) + 2) - # Test optional name def test_optional_name(self): """ @@ -99,7 +75,6 @@ def fn1(x): gc.collect() - # Every backend exposes the generated C++. self.assertTrue(hasattr(fn1, "__cpp_wrapper__")) self.assertTrue(isinstance(fn1.__cpp_wrapper__, str)) # The generated code is held by the attribute, by the registry of @@ -107,17 +82,7 @@ def fn1(x): # of gInterpreter.Declare alive on the TInterpreter instance. self.assertLessEqual(sys.getrefcount(fn1.__cpp_wrapper__), 4) - if IS_NUMBA: - # The intermediate Python wrapper and the jitted callable only - # exist in the numba implementation. - self.assertTrue(hasattr(fn1, "__py_wrapper__")) - self.assertTrue(isinstance(fn1.__py_wrapper__, str)) - self.assertEqual(sys.getrefcount(fn1.__py_wrapper__), 2) - - self.assertTrue(hasattr(fn1, "numba_func")) - self.assertEqual(sys.getrefcount(fn1.numba_func), 3) - else: - self.assertEqual(fn1.__pydeclare_cpp_name__, "Numba::fn1") + self.assertEqual(fn1.__pydeclare_cpp_name__, "Numba::fn1") # Test cling integration def test_cling(self): @@ -191,7 +156,6 @@ def square_array(x): self.assertEqual(df.Sum("x").GetValue(), 24) self.assertEqual(df.Sum("x_sq").GetValue(), 56) - @needs_cppyy_numba_ext def test_rdataframe_LorentzVector(self): """ Test function call as part of RDataFrame with ROOT::Math::LorentzVector @@ -400,7 +364,7 @@ def fn7ul(x): class NumbaDeclareArray(unittest.TestCase): """ - Test decorator to create C++ wrapper for Python callables using numba with RVecs + Test decorator to create C++ wrapper for Python callables with RVecs """ test_inputs = [default_test_inputs] @@ -673,8 +637,7 @@ def pass_reference(v): class NumbaDeclareInferred(unittest.TestCase): """ - Test decorator created with a reconstructed list of arguments using RDF column types, - and a return type inferred from the numba jitted function. + Test decorator with a return type inferred from the translated function. """ def test_fund_types(self): @@ -685,17 +648,19 @@ def test_fund_types(self): with self.subTest("function"): + @ROOT.Numba.Declare(["unsigned long"]) def is_even(x): return x % 2 == 0 - df = df.Define("is_even_x_1", is_even, ["x"]) - results = df.Take["bool"]("is_even_x_1").GetValue()[0] - self.assertEqual(results, True) + df = df.Define("is_even_x_1", "Numba::is_even(x)") + results = df.Take["bool"]("is_even_x_1").GetValue() + self.assertEqual(list(results), [True, False, True, False]) with self.subTest("lambda"): - df = df.Define("is_even_x_2", lambda x: x % 2 == 0, ["x"]) - results = df.Take["bool"]("is_even_x_2").GetValue()[0] - self.assertEqual(results, True) + ROOT.Numba.Declare(["unsigned long"], name="is_even_lambda")(lambda x: x % 2 == 0) + df = df.Define("is_even_x_2", "Numba::is_even_lambda(x)") + results = df.Take["bool"]("is_even_x_2").GetValue() + self.assertEqual(list(results), [True, False, True, False]) def test_rvec(self): """ @@ -705,15 +670,17 @@ def test_rvec(self): with self.subTest("function"): + @ROOT.Numba.Declare(["RVec"]) def square_rvec(v): return v * v - df = df.Define("square_rvec_1", square_rvec, ["x"]) + df = df.Define("square_rvec_1", "Numba::square_rvec(x)") results = df.Take["RVec"]("square_rvec_1").GetValue()[0] self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) with self.subTest("lambda"): - df = df.Define("square_rvec_2", lambda v: v * v, ["x"]) + ROOT.Numba.Declare(["RVec"], name="square_rvec_lambda")(lambda v: v * v) + df = df.Define("square_rvec_2", "Numba::square_rvec_lambda(x)") results = df.Take["RVec"]("square_rvec_2").GetValue()[0] self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) @@ -725,15 +692,17 @@ def test_std_vec(self): with self.subTest("function"): + @ROOT.Numba.Declare(["std::vector"]) def square_std_vec(v): return v * v - df = df.Define("square_std_vec_1", square_std_vec, ["x"]) + df = df.Define("square_std_vec_1", "Numba::square_std_vec(x)") results = df.Take["RVec"]("square_std_vec_1").GetValue()[0] self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) with self.subTest("lambda"): - df = df.Define("square_std_vec_2", lambda v: v * v, ["x"]) + ROOT.Numba.Declare(["std::vector"], name="square_std_vec_lambda")(lambda v: v * v) + df = df.Define("square_std_vec_2", "Numba::square_std_vec_lambda(x)") results = df.Take["RVec"]("square_std_vec_2").GetValue()[0] self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) @@ -745,15 +714,17 @@ def test_std_array(self): with self.subTest("function"): + @ROOT.Numba.Declare(["std::array"]) def square_std_arr(v): return v * v - df = df.Define("square_std_arr_1", square_std_arr, ["x"]) + df = df.Define("square_std_arr_1", "Numba::square_std_arr(x)") results = df.Take["RVec"]("square_std_arr_1").GetValue()[0] self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) with self.subTest("lambda"): - df = df.Define("square_std_arr_2", lambda v: v * v, ["x"]) + ROOT.Numba.Declare(["std::array"], name="square_std_arr_lambda")(lambda v: v * v) + df = df.Define("square_std_arr_2", "Numba::square_std_arr_lambda(x)") results = df.Take["RVec"]("square_std_arr_2").GetValue()[0] self.assertTrue(np.array_equal(results, np.array([1, 4, 9]))) @@ -761,22 +732,835 @@ def test_missing_signature(self): """ A method call on a C++ object, with no return type given. - numba cannot infer the return type here and raises. The transpiler - backends do not have to infer it: they emit the call and let the C++ - compiler deduce the type, so the same code just works. + The return type does not have to be inferred on the Python side: the + call is emitted and the C++ compiler deduces the type. """ + @ROOT.Numba.Declare(["ROOT::Math::PtEtaPhiMVector"]) def f(x): return x.M() df = ROOT.RDataFrame(4).Define("v", "ROOT::Math::PtEtaPhiMVector(1, 2, 3, 4)") + masses = df.Define("m", "Numba::f(v)").Take["double"]("m").GetValue() + self.assertAlmostEqual(masses[0], 4.0) + + +class PyDeclareNamespace(unittest.TestCase): + """The ROOT.Py.Declare spelling, and what the translation adds.""" + + def test_py_namespace(self): + """ + The generated function lands in the Py namespace + """ + + @ROOT.Py.Declare(["float"], "float") + def py_double(x): + return 2.0 * x + + self.assertEqual(ROOT.Py.py_double(21.0), 42.0) + self.assertEqual(py_double.__pydeclare_cpp_name__, "Py::py_double") + self.assertIn("namespace Py", py_double.__cpp_wrapper__) + + def test_control_flow(self): + """ + if/elif/else, for over an array, while, and early returns + """ + + @ROOT.Py.Declare(["double"], "double") + def clamped(x): + if x < 0.0: + return 0.0 + elif x > 10.0: + return 10.0 + return x + + self.assertEqual([ROOT.Py.clamped(v) for v in (-5.0, 5.0, 50.0)], [0.0, 5.0, 10.0]) + + @ROOT.Py.Declare(["RVecD", "double"], "long") + def count_above(v, thr): + n = 0 + for x in v: + if x > thr: + n = n + 1 + return n + + self.assertEqual(ROOT.Py.count_above(ROOT.RVecD([1.0, 5.0, 9.0]), 4.0), 2) + + @ROOT.Py.Declare(["long"], "double") + def harmonic(n): + total = 0.0 + for i in range(1, n + 1): + total += 1.0 / i + return total + + expected = 0.0 + for i in range(1, 5): + expected += 1.0 / i + self.assertAlmostEqual(ROOT.Py.harmonic(4), expected, places=15) + + @ROOT.Py.Declare(["RVecD"], "double") + def first_negative_or_sum(v): + i = 0 + while i < v.size: + if v[i] < 0.0: + return v[i] + i += 1 + return v.sum() + + self.assertEqual(ROOT.Py.first_negative_or_sum(ROOT.RVecD([1.0, -2.0, 3.0])), -2.0) + self.assertEqual(ROOT.Py.first_negative_or_sum(ROOT.RVecD([1.0, 2.0, 3.0])), 6.0) + + def test_python_semantics(self): + """ + The operators where Python and C++ disagree keep Python's meaning + """ + + @ROOT.Py.Declare(["int", "int"], "double") + def true_division(a, b): + return a / b + + @ROOT.Py.Declare(["int", "int"], "int") + def floor_division(a, b): + return a // b + + @ROOT.Py.Declare(["int", "int"], "int") + def modulo(a, b): + return a % b + + @ROOT.Py.Declare(["RVecD"], "double") + def last(v): + return v[-1] + + self.assertEqual(ROOT.Py.true_division(7, 2), 7 / 2) + self.assertEqual(ROOT.Py.floor_division(-7, 2), -7 // 2) + self.assertEqual(ROOT.Py.modulo(-7, 2), -7 % 2) + self.assertEqual(ROOT.Py.last(ROOT.RVecD([1.0, 2.0, 3.0])), 3.0) + + def test_cpp_interoperability(self): + """ + Translated code can name anything cling knows, and other declarations + """ + + @ROOT.Py.Declare(["double"], "double") + def scaled(x): + return 2.5 * x + + @ROOT.Py.Declare(["double"], "double") + def combined(x): + return ROOT.TMath.Abs(scaled(x)) + ROOT.TMath.Sqrt(4.0) + + self.assertEqual(ROOT.Py.combined(-2.0), 7.0) + self.assertIn("TMath::Abs", combined.__cpp_wrapper__) + self.assertIn("Py::scaled", combined.__cpp_wrapper__) + + def test_constants_are_frozen(self): + """ + Values from the enclosing scope are inlined when the function is declared + """ + scale = 0.5 + + @ROOT.Py.Declare(["double"], "double") + def uses_closure(x): + return x * scale + + self.assertEqual(ROOT.Py.uses_closure(4.0), 2.0) + self.assertIn("0.5", uses_closure.__cpp_wrapper__) + self.assertNotIn("scale", uses_closure.__cpp_wrapper__) + + # The value is read when the function is declared, not cached per name: + # a second function sees the rebound value. + scale = 100.0 + + @ROOT.Py.Declare(["double"], "double") + def uses_closure_again(x): + return x * scale + + self.assertEqual(ROOT.Py.uses_closure_again(4.0), 400.0) + self.assertEqual(ROOT.Py.uses_closure(4.0), 2.0) + + def test_generated_code_is_available(self): + """ + The C++ can be read back, which is the debugging story + """ + + @ROOT.Py.Declare(["std::vector"], "std::vector") + def squared_vector(x): + return x * x + + code = squared_vector.__cpp_wrapper__ + self.assertIn("std::vector squared_vector(const std::vector &x)", code) + # std::vector is viewed as an RVec so that the operators are available + self.assertIn("AsRVec", code) + + +class PyDeclareRefusals(unittest.TestCase): + """Anything outside the supported subset has to be refused, not approximated. + + Each case below would otherwise be translated to C++ that compiles and + quietly computes something else, which is the failure mode this feature + cannot afford. + """ + + counter = 0 + + def load(self, source): + """Write a callable to a real file, so that its source can be read back. + + The translation reads the Python source in order to quote the offending + line, so the callables under test cannot come from exec(). + """ + directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, directory, True) + PyDeclareRefusals.counter += 1 + name = "pydeclare_refusal_{}".format(PyDeclareRefusals.counter) + path = os.path.join(directory, name + ".py") + with open(path, "w") as out: + out.write(textwrap.dedent(source)) + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.f, path + + def assertRefused(self, source, input_types, return_type, expected): + """Declare the callable and require a clear refusal mentioning `expected`.""" + func, _ = self.load(source) + with self.assertRaises(PyDeclareError) as caught: + ROOT.Py.Declare(input_types, return_type, name="refused")(func) + self.assertIn(expected, str(caught.exception)) + + def test_unsupported_statements(self): + self.assertRefused( + "def f(x):\n try:\n return x\n except Exception:\n return 0.0\n", + ["double"], + "double", + "'Try' is not supported", + ) + + def test_comprehension(self): + self.assertRefused( + "def f(v):\n return sum([x * x for x in v])\n", + ["RVecD"], + "double", + "List comprehensions are not supported", + ) + + def test_string_value(self): + self.assertRefused( + "def f(x):\n s = 'hello'\n return x\n", + ["double"], + "double", + "String values are not supported", + ) + + def test_truth_of_an_array(self): + self.assertRefused( + "def f(v):\n if v:\n return 1.0\n return 0.0\n", + ["RVecD"], + "double", + "truth value of an array is ambiguous", + ) + + def test_variable_changing_type(self): + self.assertRefused( + "def f(v):\n a = v.sum()\n a = v\n return 1.0\n", + ["RVecD"], + "double", + "changes type", + ) + + def test_arbitrary_python_module(self): + self.assertRefused( + "import os\n\n\ndef f(x):\n return os.getpid() * x\n", + ["double"], + "double", + "cannot be used in translated code", + ) + + def test_unknown_cpp_name(self): + self.assertRefused( + "import ROOT\n\n\ndef f(x):\n return ROOT.TMath.NoSuchFunction(x)\n", + ["double"], + "double", + "has no member 'NoSuchFunction'", + ) + + def test_error_points_at_the_source_line(self): + """The message has to say where, not just what.""" + func, path = self.load("def f(x):\n d = {1: 2}\n return x\n") + with self.assertRaises(PyDeclareError) as caught: + ROOT.Py.Declare(["double"], "double", name="refused_with_location")(func) + message = str(caught.exception) + self.assertIn(path, message) + self.assertIn("line 2", message) + self.assertIn("d = {1: 2}", message) + # ... and name the construct it actually found. + self.assertIn("Dict literals", message) + + def test_random_numbers(self): + """The most important refusal: a frozen draw would be silently wrong. + + np.random.normal() does not depend on the arguments, so constant + folding would happily evaluate it once and inline the result, giving + every entry in the dataset the same "random" number. + """ + self.assertRefused( + "import numpy as np\n\n\ndef f(x):\n return np.random.normal() * x\n", + ["double"], + "double", + "'np.random' is not available", + ) + self.assertRefused( + "import numpy as np\n\n\ndef f(x):\n return x + np.random.rand()\n", + ["double"], + "double", + "'np.random' is not available", + ) + + def test_reading_a_file_at_declaration_time(self): + self.assertRefused( + "import numpy as np\n\n\ndef f(x):\n return x + float(np.load('weights.npy'))\n", + ["double"], + "double", + "'np.load' is not available", + ) + + def test_sets_and_tuples(self): + self.assertRefused( + "def f(x):\n s = {1, 2}\n return x\n", ["double"], "double", "Set literals" + ) + self.assertRefused( + "def f(x):\n t = (x, x)\n return t[0]\n", ["double"], "double", "Tuple literals" + ) + + def test_lambda_in_the_body(self): + self.assertRefused( + "def f(x):\n g = lambda y: y * 2\n return g(x)\n", + ["double"], + "double", + "Nested lambdas are not supported", + ) + + def test_with_and_global(self): + self.assertRefused( + "def f(x):\n with open('f') as fh:\n return x\n", + ["double"], + "double", + "'With' is not supported", + ) + self.assertRefused( + "def f(x):\n global g\n return x\n", ["double"], "double", "'Global' is not supported" + ) + + def test_f_strings_and_generators(self): + self.assertRefused( + "def f(x):\n return float(len(f'{x}'))\n", ["double"], "double", "f-strings are not supported" + ) + self.assertRefused( + "def f(v):\n return sum(x for x in v)\n", ["RVecD"], "double", "comprehensions are not supported" + ) + + def test_star_args(self): + self.assertRefused( + "def f(*args):\n return args[0]\n", ["double"], "double", "*args" + ) + + def test_multi_dimensional_indexing(self): + self.assertRefused( + "def f(v):\n return v[0, 1]\n", ["RVecD"], "double", "Multi-dimensional indexing" + ) + + def test_signature_mismatch(self): + self.assertRefused( + "def f(x, y):\n return x + y\n", ["double"], "double", "takes 2 argument" + ) + + def test_mutating_an_array_that_python_would_alias(self): + """'w = v' aliases in Python and copies here, so writing through it + would mean two different things.""" + self.assertRefused( + "def f(v):\n w = v\n w[0] = 100.0\n return v[0]\n", + ["RVecD"], + "double", + "Cannot assign to an element of 'w'", + ) + self.assertRefused( + "def f(v):\n w = v[1:]\n w[0] = 100.0\n return v[1]\n", + ["RVecD"], + "double", + "Cannot assign to an element of 'w'", + ) + + def test_reading_a_variable_that_may_be_unbound(self): + """Python raises UnboundLocalError; a zero would look like a result.""" + self.assertRefused( + "def f(x):\n if x > 0:\n y = 1.5\n return y\n", + ["double"], + "double", + "only assigned inside a nested block", + ) + + def test_conditional_expression_with_an_array_condition(self): + """numpy raises on this too; np.where is the element-wise form.""" + self.assertRefused( + "def f(v):\n return v if v > 0 else -v\n", + ["RVecD"], + "ROOT::RVec", + "The truth value of an array is ambiguous", + ) + + def test_non_integer_slice_bounds(self): + self.assertRefused( + "def f(v, x):\n return v[x:]\n", + ["RVecD", "double"], + "ROOT::RVec", + "Slice indices must be integers", + ) + + def test_assigning_to_an_array_parameter(self): + self.assertRefused( + "import numpy as np\n\n\ndef f(v):\n v = np.array([1.0])\n return v[0]\n", + ["RVecD"], + "double", + "Cannot assign to the array parameter 'v'", + ) + + +class PyDeclareHasNoNumbaDependency(unittest.TestCase): + """The translation is the point: nothing here may pull in numba.""" + + def test_numba_is_not_imported(self): + for module in ("numba", "llvmlite", "cffi"): + self.assertNotIn(module, sys.modules, "{} was imported by ROOT.Py.Declare".format(module)) + + +class PyDeclareAgreesWithPython(unittest.TestCase): + """The translated function has to give what the plain Python one gives. + + Every case here computes its reference by calling the undecorated callable, + so the assertion is against Python itself rather than against a constant + that was read off this implementation. + """ + + counter = 0 + + def check(self, func, input_types, return_type, cases, places=12): + """Declare *func* and compare it with the Python original on *cases*.""" + PyDeclareAgreesWithPython.counter += 1 + name = "agree_{}".format(PyDeclareAgreesWithPython.counter) + ROOT.Py.Declare(input_types, return_type, name=name)(func) + translated = getattr(ROOT.Py, name) + for args in cases: + with self.subTest(func=func.__name__, args=args): + expected = func(*args) + got = translated(*[_as_cpp(a) for a in args]) + if isinstance(expected, np.ndarray): + self.assertEqual(list(got), list(expected)) + elif isinstance(expected, (float, np.floating)): + self.assertAlmostEqual(got, float(expected), places=places) + else: + self.assertEqual(got, expected) + # An integer result has to stay integral: Python's '2 ** 10' + # is an int, and a numpy integer is one too. + self.assertIsInstance(got, (int, bool)) + + # -- the documented Python/C++ divergences ------------------------------ + + def test_true_division(self): + def truediv(a, b): + return a / b + + self.check(truediv, ["int", "int"], "double", [(7, 2), (-7, 2), (1, 8)]) + + def test_floor_division(self): + def floordiv(a, b): + return a // b + + self.check(floordiv, ["int", "int"], "long", [(7, 2), (-7, 2), (7, -2), (-7, -2)]) + + def test_modulo_takes_the_sign_of_the_divisor(self): + def mod(a, b): + return a % b + + self.check(mod, ["int", "int"], "long", [(7, 3), (-7, 3), (7, -3), (-7, -3)]) + + def test_power_of_two_integers_stays_integral(self): + def ipow(a, b): + return a**b + + self.check(ipow, ["int", "int"], "long", [(2, 10), (3, 0), (-2, 3), (10, 6)]) + + def test_power_accumulates_in_a_wide_integer(self): + """A short base must not overflow at 2**15, as Python's int does not.""" + + def wideipow(x): + return x**3 + + self.check(wideipow, ["short"], "long", [(100,), (2,), (-100,)]) + + def test_power_of_floats(self): + def fpow(a, b): + return a**b + + self.check(fpow, ["double", "double"], "double", [(2.0, 0.5), (9.0, -1.0)]) + + def test_round_is_half_to_even(self): + def myround(x): + return round(x) + + self.check(myround, ["double"], "long", [(0.5,), (1.5,), (2.5,), (-0.5,), (-1.5,), (3.7,)]) + + def test_negative_index(self): + def last(v): + return v[-1] + + self.check(last, ["RVecD"], "double", [(np.array([1.0, 2.0, 3.0]),)]) + + def test_sum_of_booleans_is_an_integer(self): + def countpos(v): + return (v > 0).sum() + + self.check(countpos, ["RVecD"], "long", [(np.array([1.0, -2.0, 3.0, -4.0]),)]) + + def test_sum_of_narrow_integers_does_not_overflow(self): + """numpy accumulates into a wide integer; the element type would wrap.""" + + def total(v): + return v.sum() + + self.check(total, ["ROOT::RVec"], "long", [(np.array([3] * 100, dtype=np.int8),)]) + + # -- slicing ------------------------------------------------------------ + + def test_slices(self): + cases = [(np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]),)] + for i, body in enumerate( + [ + "v[1:4]", + "v[-2:]", + "v[:3]", + "v[::2]", + "v[1::2]", + "v[10:20]", + "v[4:1:-1]", + "v[::-1]", + "v[-100:100]", + "v[3:1]", + ] + ): + namespace = {"np": np} + exec("def sliced(v):\n return {}\n".format(body), namespace) # noqa: S102 + func = namespace["sliced"] + func.__name__ = "sliced_{}".format(i) + # exec'd source is unreadable, so go through a real file instead. + with self.subTest(slice=body): + self.check(_reload_from_file(func, body), ["RVecD"], "ROOT::RVec", cases) + + # -- the parts of the subset that had no test at all -------------------- + + def test_builtins(self): + def usebuiltins(v, x): + return float(len(v)) + max(x, 2.0) + min(x, 2.0) + abs(x) + float(sum(v)) + pow(x, 2.0) + + self.check(usebuiltins, ["RVecD", "double"], "double", [(np.array([1.0, 2.0]), -3.0)]) + + def test_reductions(self): + def reductions(v): + return v.mean() + v.std() + v.min() + v.max() + float(v.argmin() + v.argmax()) + v.prod() + + self.check(reductions, ["RVecD"], "double", [(np.array([3.0, 1.0, 2.0]),)]) + + def test_any_and_all(self): + def anyall(v): + return int((v > 0).any()) * 10 + int((v > 0).all()) + + self.check(anyall, ["RVecD"], "long", [(np.array([1.0, -1.0]),), (np.array([1.0, 2.0]),)]) + + def test_np_where(self): + def clip(v): + return np.where(v > 0, v, 0.0) + + self.check(clip, ["RVecD"], "ROOT::RVec", [(np.array([1.0, -2.0, 3.0]),)]) + + def test_elementwise_maths_and_constants(self): + def maths(x): + return np.sqrt(abs(x)) + np.sin(x) + np.log(abs(x) + 1.0) + np.pi + + self.check(maths, ["double"], "double", [(2.0,), (-3.0,)]) + + def test_chained_comparison(self): + def between(x): + return 0.0 < x < 10.0 + + self.check(between, ["double"], "bool", [(5.0,), (-1.0,), (20.0,)]) + + def test_and_or_evaluate_to_an_operand(self): + """Python's 'or' gives back a value, not a bool: '0.0 or 5.0' is 5.0.""" + + def orelse(x, y): + return x or y + + self.check(orelse, ["double", "double"], "double", [(0.0, 5.0), (3.0, 5.0)]) + + def andalso(x, y): + return x and y + + self.check(andalso, ["double", "double"], "double", [(0.0, 5.0), (3.0, 5.0)]) + + def test_conditional_expression(self): + def sign(x): + return 1.0 if x > 0 else -1.0 + + self.check(sign, ["double"], "double", [(2.0,), (-2.0,)]) + + def test_bitwise_and_shifts(self): + def bits(a, b): + return (a & b) + (a | b) + (a ^ b) + (a << 2) + (a >> 1) + ~a + + self.check(bits, ["int", "int"], "long", [(12, 10), (1, 0)]) + + def test_break_continue_and_assert(self): + def loop(v): + total = 0.0 + for x in v: + if x < 0: + continue + if x > 100.0: + break + assert x == x + total += x + return total + + self.check(loop, ["RVecD"], "double", [(np.array([1.0, -2.0, 3.0, 200.0, 5.0]),)]) + + def test_annotated_assignment(self): + def annotated(v): + total: "double" = 0.0 # noqa: F821 -- a C++ type, not a Python name + for x in v: + total += x + return total + + self.check(annotated, ["RVecD"], "double", [(np.array([1.0, 2.0, 3.0]),)]) + + def test_mask_indexing(self): + def positives(v): + return v[v > 0].sum() + + self.check(positives, ["RVecD"], "double", [(np.array([1.0, -2.0, 3.0]),)]) + + def test_loop_variable_survives_the_loop(self): + """Python's for does not scope its variable to the loop body.""" + + def leaks(n): + i = 0 + if n > 0: + for i in range(n): + pass + return i + + self.check(leaks, ["int"], "long", [(5,), (0,)]) + + def test_unary_minus_does_not_fuse(self): + """'-(-x)' must not come out as the decrement operator.""" + + def negneg(x): + return -(-x) + +(+x) + + self.check(negneg, ["long"], "long", [(5,), (-3,)]) + + def test_parameter_can_be_reassigned(self): + def absval(x): + if x < 0: + x = -x + return x + + self.check(absval, ["double"], "double", [(-3.0,), (3.0,)]) + + def test_element_assignment_on_a_locally_built_array(self): + def build(x): + a = np.array([x, x + 1.0]) + a[0] = 99.0 + return a + + self.check(build, ["double"], "ROOT::RVec", [(1.0,)]) + + +class PyDeclareMixedSignedness(unittest.TestCase): + """Python has no unsigned integers, so C++'s conversions are wrong for us. + + ``nMuon`` and friends are unsigned columns, so this is reachable from + ordinary RDataFrame code rather than being a corner case. + """ + + def check(self, name, func, input_types, return_type, args, expected): + ROOT.Py.Declare(input_types, return_type, name=name)(func) + self.assertEqual(getattr(ROOT.Py, name)(*args), expected) + + def test_modulo(self): + def mixmod(a, b): + return a % b + + self.check("mix_mod", mixmod, ["int", "unsigned int"], "long", (-7, 3), -7 % 3) + + def test_floor_division(self): + def mixfdiv(a, b): + return a // b + + self.check("mix_fdiv", mixfdiv, ["int", "unsigned int"], "long", (-7, 3), -7 // 3) + + def test_unary_minus_of_an_unsigned_value(self): + def negu(x): + return -x + + self.check("mix_neg", negu, ["unsigned int"], "long", (3,), -3) + + def test_comparison(self): + def gt(a, b): + return a > b + + self.check("mix_gt", gt, ["int", "unsigned int"], "bool", (-1, 2), -1 > 2) + + def test_max(self): + def mymax(a, b): + return max(a, b) + + self.check("mix_max", mymax, ["int", "unsigned int"], "long", (-1, 2), max(-1, 2)) + + def test_min(self): + def mymin(a, b): + return min(a, b) + + self.check("mix_min", mymin, ["int", "unsigned int"], "long", (-1, 2), min(-1, 2)) + + def test_abs_of_an_unsigned_value(self): + """std::abs has no unsigned overload; the call would be ambiguous.""" + + def myabs(x): + return abs(x) + + self.check("mix_abs", myabs, ["unsigned int"], "unsigned int", (3,), 3) + + +class PyDeclareRaisesInsteadOfMisbehaving(unittest.TestCase): + """Where Python raises, the translation must raise too. + + Every case here used to either abort the process or read out of bounds, + which in an event loop means a crash or a silently wrong number. + """ + + def declare(self, name, func, input_types, return_type): + ROOT.Py.Declare(input_types, return_type, name=name)(func) + return getattr(ROOT.Py, name) - if IS_NUMBA: - with self.assertRaises(Exception): - df.Define("m", f, ["v"]) - else: - masses = df.Define("m", f, ["v"]).Take["double"]("m").GetValue() - self.assertAlmostEqual(masses[0], 4.0) + def test_integer_division_by_zero(self): + def fdiv(a, b): + return a // b + + with self.assertRaises(Exception): + self.declare("raise_fdiv", fdiv, ["int", "int"], "long")(7, 0) + + def test_modulo_by_zero(self): + def mod(a, b): + return a % b + + with self.assertRaises(Exception): + self.declare("raise_mod", mod, ["int", "int"], "long")(7, 0) + + def test_true_division_of_integers_by_zero(self): + def div(a, b): + return a / b + + with self.assertRaises(Exception): + self.declare("raise_div", div, ["int", "int"], "double")(7, 0) + + def test_index_past_the_end(self): + def at(v, i): + return v[i] + + fn = self.declare("raise_at", at, ["RVecD", "int"], "double") + with self.assertRaises(Exception): + fn(ROOT.VecOps.RVec("double")([1.0, 2.0, 3.0]), 100) + with self.assertRaises(Exception): + fn(ROOT.VecOps.RVec("double")([1.0, 2.0, 3.0]), -100) + + def test_index_past_the_end_with_an_unsigned_index(self): + def atu(v, i): + return v[i] + + fn = self.declare("raise_atu", atu, ["RVecD", "unsigned int"], "double") + with self.assertRaises(Exception): + fn(ROOT.VecOps.RVec("double")([1.0, 2.0, 3.0]), 100) + + def test_index_with_a_constant_past_the_end(self): + def at5(v): + return v[5] + + fn = self.declare("raise_at5", at5, ["RVecD"], "double") + with self.assertRaises(Exception): + fn(ROOT.VecOps.RVec("double")([1.0, 2.0, 3.0])) + + def test_where_with_branches_shorter_than_the_condition(self): + def wh(a, b): + return np.where(a > 0, b, 0.0) + + fn = self.declare("raise_where", wh, ["RVecD", "RVecD"], "ROOT::RVec") + with self.assertRaises(Exception): + fn(ROOT.VecOps.RVec("double")([1.0] * 4), ROOT.VecOps.RVec("double")([5.0, 6.0])) + + +class PyDeclareTypeSpellings(unittest.TestCase): + """Every spelling of an RVec that ROOT accepts elsewhere has to work here.""" + + def test_rvec_alias_spellings(self): + spellings = [ + "RVecF", + "ROOT::RVecF", + "ROOT::VecOps::RVecF", + "RVec", + "ROOT::RVec", + "ROOT::VecOps::RVec", + ] + for i, spelling in enumerate(spellings): + with self.subTest(spelling=spelling): + + def total(v): + return v.sum() + + name = "spelling_{}".format(i) + ROOT.Py.Declare([spelling], "double", name=name)(total) + self.assertAlmostEqual( + getattr(ROOT.Py, name)(ROOT.VecOps.RVec("float")([1.0, 2.0, 3.0])), 6.0 + ) + + +def _as_cpp(value): + """numpy arrays go in as RVecs; everything else is passed straight through.""" + if isinstance(value, np.ndarray): + return ROOT.VecOps.RVec(_CPP_OF_DTYPE[value.dtype.name])(value) + return value + + +_CPP_OF_DTYPE = { + "float64": "double", + "float32": "float", + "int8": "signed char", + "int32": "int", + "int64": "long long", + "bool": "bool", +} + + +def _reload_from_file(func, body): + """Re-create *func* in a real file, so that its source can be read.""" + directory = tempfile.mkdtemp() + module_name = "pydeclare_slice_{}".format(func.__name__) + path = os.path.join(directory, module_name + ".py") + with open(path, "w") as out: + out.write("import numpy as np\n\n\ndef sliced(v):\n return {}\n".format(body)) + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.sliced.__name__ = func.__name__ + return module.sliced if __name__ == "__main__": diff --git a/bindings/pyroot/pythonizations/test/rdf_define_pyz.py b/bindings/pyroot/pythonizations/test/rdf_define_pyz.py index debefbc9a9750..32689ac0cf9d9 100644 --- a/bindings/pyroot/pythonizations/test/rdf_define_pyz.py +++ b/bindings/pyroot/pythonizations/test/rdf_define_pyz.py @@ -1,13 +1,9 @@ import multiprocessing import unittest -import numba # noqa: F401 import numpy as np import ROOT -# numba is not used directly, but tests can crash when ROOT is built with -# builtin_llvm=OFF and numba is not imported at the beginning - # C++ type names, used as type annotations of the Python callables passed to Define DOUBLE = "double" ULONG64 = "ULong64_t" @@ -49,11 +45,11 @@ class PyDefine(unittest.TestCase): def test_with_dtypes(self): """ - Tests the pythonized define with all the numba declare datatypes and + Tests the pythonized define with all the supported datatypes and """ - numba_declare_dtypes = ["float", "double", "int", "unsigned int", "long", "unsigned long", "bool"] + declare_dtypes = ["float", "double", "int", "unsigned int", "long", "unsigned long", "bool"] rdf = ROOT.RDataFrame(10) - for type in numba_declare_dtypes: + for type in declare_dtypes: col_name = "col_" + type.replace(" ", "") rdf = rdf.Define(col_name, f"({type}) rdfentry_") @@ -64,12 +60,16 @@ def to_array(col: type) -> f"ROOT::VecOps::RVec<{type}>": arr = np.arange(0, 10) if type == "bool": arr = np.array(arr, dtype="bool") - flag1 = np.array_equal(rdf.AsNumpy()[col_name], arr) - flag2 = True + self.assertTrue( + np.array_equal(rdf.AsNumpy()[col_name], arr), + "scalar column differs for {}".format(type), + ) for idx, entry in enumerate(rdf.AsNumpy()[col_name + "_arr"]): - if not (entry[0] == arr[idx] and entry[1] == arr[idx]): - flag2 = False - self.assertTrue(flag1 and flag2) + self.assertEqual( + [entry[0], entry[1]], + [arr[idx], arr[idx]], + "array column differs for {} at entry {}".format(type, idx), + ) def test_define_overload1(self): rdf = ROOT.RDataFrame(10).Define("x", "rdfentry_") diff --git a/bindings/pyroot/pythonizations/test/rdf_filter_pyz.py b/bindings/pyroot/pythonizations/test/rdf_filter_pyz.py index edfcc181d3f37..cbf10dea54c59 100755 --- a/bindings/pyroot/pythonizations/test/rdf_filter_pyz.py +++ b/bindings/pyroot/pythonizations/test/rdf_filter_pyz.py @@ -1,14 +1,10 @@ import os import unittest -import numba # noqa: F401 import numpy as np import ROOT from rdf_filter_pyz_helper import TYPE_TO_SYMBOL, CreateData, filter_dict -# numba is not used directly, but tests can crash when ROOT is built with -# builtin_llvm=OFF and numba is not imported at the beginning - # C++ type name, used as type annotation of the Python callables passed to Filter DOUBLE = "double" @@ -27,7 +23,10 @@ def test_with_dtypes(self): test_cols = [str(c) for c in rdf.GetColumnNames()] for col_name in test_cols: func = filter_dict[TYPE_TO_SYMBOL[col_name]] # filter function - x = rdf.Mean(col_name).GetValue() + # A threshold that occurs in the data, so that '>' and '>=' give + # different answers and the cross-check can tell them apart. The + # column mean is almost never an actual entry. + x = rdf.Take[col_name](col_name).GetValue()[0] if col_name == "Bool_t": x = True diff --git a/bindings/pyroot/pythonizations/test/rdf_filter_pyz_helper.py b/bindings/pyroot/pythonizations/test/rdf_filter_pyz_helper.py index 367fbd963d131..b1d6dfddb5849 100755 --- a/bindings/pyroot/pythonizations/test/rdf_filter_pyz_helper.py +++ b/bindings/pyroot/pythonizations/test/rdf_filter_pyz_helper.py @@ -1,36 +1,52 @@ import ROOT -import numpy as np + + +def _declare_random_helper(): + """Declare a helper giving a random integer over a signed or unsigned range. + + gRandom->Integer takes a UInt_t bound, so it can neither produce the signed + ranges the Short_t and Int_t columns need nor span the full 2**32 width of + the Int_t one. Scaling Rndm(), which returns a double in ]0, 1[, does + both, and draws from [low, high) like Python's range -- Uniform() returns + ]x1, x2], which would never produce `low` and could overflow at `high`. + + gRandom is a TRandom3 with its default seed, so the dataset is the same on + every run. That is wanted here -- these tests compare a translated filter + against the equivalent C++ one, and a reproducible dataset is worth more + than a fresh one. + """ + if hasattr(ROOT, "RandomInRange"): + return + ROOT.gInterpreter.Declare(""" + Long64_t RandomInRange(Long64_t low, Long64_t high) + { + const Double_t width = static_cast(high - low); + return low + static_cast(gRandom->Rndm() * width); + } + """) + def CreateData(): """ This function generates the root files of various datatypes with random values to test them. Datatypes could be generated are Strings, Char_t, UChar_t """ - # function to create random numbers.. gRandom did not give me signed integers - @ROOT.Numba.Declare(['int', 'bool'], 'long') - def random_long(bits, signed): - if signed: - low = -1*2**(bits - 1) - high = 2**(bits - 1) -1 - else: - low = 0 - high = 2**bits - return np.random.randint(low, high) - + _declare_random_helper() + N = 100 # df with 100 entries df = ROOT.RDataFrame(N) col_name = "Short_t" - df = df.Define(col_name, f"({col_name}) Numba::random_long(16, true)") + df = df.Define(col_name, f"({col_name}) RandomInRange(-32768, 32768)") col_name = "UShort_t" - df = df.Define(col_name, f"({col_name}) Numba::random_long(16, false)") + df = df.Define(col_name, f"({col_name}) RandomInRange(0, 65536)") col_name = "Int_t" - df = df.Define(col_name, f"({col_name}) Numba::random_long(32, true)") + df = df.Define(col_name, f"({col_name}) RandomInRange(-2147483648LL, 2147483648LL)") col_name = "UInt_t" - df = df.Define(col_name, f"({col_name}) Numba::random_long(32, false)") + df = df.Define(col_name, f"({col_name}) RandomInRange(0, 4294967296LL)") col_name = "Float_t" df = df.Define(col_name, f"({col_name}) gRandom->Gaus()") From bcf019f8127f4c6a75dca376fe90d554fc08cb91 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 31 Aug 2026 05:26:23 +0000 Subject: [PATCH 6/8] [Python] Drop numba and cffi as dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in ROOT's own Python code imports them any more: _numbadeclare.py was the only user of both, and the RDataFrame pythonizations reach the translation directly. Removes them from requirements.txt and the wheel CI requirements, drops the requires_numba tutorial fixture, renames TREE_TO_NUMBA now that the map has nothing to do with numba, and updates the RDataFrame documentation and the df038/df101 tutorials to describe what actually happens. ROOT.NumbaExt is deliberately left alone. It is the other direction -- numba-compiled Python calling into C++ through cppyy's numba extension -- and is a thin re-export of upstream cppyy, imported only when a user asks for it. Removing that is a separate decision. 🤖 Done with the help of AI --- README/ReleaseNotes/v644/index.md | 51 +++++++++++++++++++ .../python/ROOT/_pythonization/_rdataframe.py | 30 ++++++----- .../_pythonization/_rdf_conversion_maps.py | 4 +- .../python/ROOT/_pythonization/_rdf_pyz.py | 8 +-- cmake/scripts/CMakeCPackOptions.cmake.in | 2 +- requirements.txt | 5 +- test/wheels/requirements-ci.txt | 4 -- tutorials/CMakeLists.txt | 6 +-- .../analysis/dataframe/df038_NumbaDeclare.py | 21 +++++--- .../analysis/dataframe/df101_h1Analysis.py | 2 +- 10 files changed, 94 insertions(+), 39 deletions(-) diff --git a/README/ReleaseNotes/v644/index.md b/README/ReleaseNotes/v644/index.md index dde65790e5f56..d3bae490ddf70 100644 --- a/README/ReleaseNotes/v644/index.md +++ b/README/ReleaseNotes/v644/index.md @@ -28,6 +28,7 @@ The following people have contributed to this new version: * The TMVA deep learning methods `MethodDL` (`TMVA::Types::kDL`) and `MethodDNN` (`TMVA::Types::kDNN`), together with all classes in the `TMVA::DNN` namespace and their helper classes `TMVA::Monitoring` and `Pattern`, are deprecated and will be removed in ROOT 6.46. Neural networks are nowadays trained with dedicated machine learning libraries, so maintaining a separate implementation with its own CPU, CUDA and cuDNN backends is no longer justified. Users are encouraged to train their models directly with these libraries, which integrate well with ROOT via the `ROOT.Experimental.ML.RDataLoader`, and to use SOFIE for inference in C++. The `TMVA_CNN_Classification` and `TMVA_RNN_Classification` tutorials are removed already, and the other TMVA tutorials do not use these methods anymore. We are aware that models trained with these methods may be stored in TMVA XML weight files and still be evaluated via `TMVA::Reader` in production workflows. If you need to keep running such models after ROOT 6.46, please reach out to the ROOT team by opening a [GitHub issue](https://github.com/root-project/root/issues) or posting on the [ROOT forum](https://root-forum.cern.ch/), so that a migration path to SOFIE can be provided. * The build options `tmva-cpu`, `tmva-gpu`, `tmva-cudnn` and `use_gsl_cblas` only affect the deprecated TMVA deep learning methods, so they are deprecated as well and will be removed in ROOT 6.46. The `tmva-cpu` option is now `OFF` by default, also with `-Dall=ON`, so `MethodDL` runs on a single thread without BLAS and `MethodDNN` cannot be trained, unless ROOT is configured with `-Dtmva-cpu=ON`. Evaluating existing weight files with `TMVA::Reader` works in both cases. +* `numba` and `cffi` are no longer dependencies of PyROOT. `ROOT.Numba.Declare` translates the Python callable to C++ instead of compiling it with numba; see the Python Interface section. ## Build system @@ -42,6 +43,56 @@ RenderCore, the WebGL engine of the REve client, is fetched at build time as the ## Python Interface +### `ROOT.Numba.Declare` no longer uses numba + +The decorator now translates the decorated Python callable to C++ source and declares that to the interpreter, instead +of compiling it with numba and calling into it through a function pointer. The spelling and the `Numba` C++ namespace +are unchanged, so existing code such as + +~~~{.py} +@ROOT.Numba.Declare(["float", "int"], "float") +def pypow(x, y): + return x**y + +df.Define("y", "Numba::pypow(x, 3)") +~~~ + +keeps working, and numba and cffi are no longer dependencies of PyROOT. + +The same decorator is available as `ROOT.Py.Declare`, which declares into the `Py` namespace and is the recommended +spelling for new code. Because the result is ordinary C++ rather than an opaque address, it is inlined into the +RDataFrame event loop, it can call anything the interpreter already knows -- methods on ROOT classes, functions such +as `TMath::Abs`, other declared callables -- and the generated code can be read back from the callable as +`__cpp_wrapper__`. + +Practical consequences: + +* Callables returning an `RVec`, `std::vector` or `std::array` no longer allocate and copy the result twice per entry. +* Declaring a function is roughly an order of magnitude faster, which is noticeable in notebooks. +* Methods on C++ objects, for example `ROOT::Math::PtEtaPhiMVector::M()`, work on every platform rather than only + where the experimental cppyy/numba bridge is tested. + +Only a subset of Python is translated. Anything outside it is refused when the function is declared, with the +offending source line quoted, rather than being approximated in C++. Constructs that numba accepted and this +translation does not include comprehensions, tuple unpacking, dictionaries, sets, strings, `try`, `with`, lambdas in +the function body, array allocation such as `np.zeros`, calls to other plain Python functions, and any third-party +call -- `np.random` in particular, because no C++ generator reproduces numpy's stream and silently different random +numbers are worse than an error. The supported subset is documented in the RDataFrame Python section and in +`bindings/pyroot/pythonizations/python/ROOT/_pydeclare/README.md` in the source tree. + +Two further differences from the numba implementation are worth knowing about when migrating: + +* The translation reads the Python source of the callable with `inspect.getsource`, so a callable whose source is not + available cannot be translated. In practice this means functions typed at the plain `python` prompt, passed to + `python -c`, piped in on standard input, or built with `exec()`. Functions in a file, in a Jupyter or IPython cell, + and in a notebook all work. +* The decorated callable no longer carries the `numba_func`, `__py_wrapper__` and `__numba_cfunc__` attributes. The + generated C++ is available instead as `__cpp_wrapper__`. + +Note that `ROOT.NumbaExt`, which enables numba-compiled Python to call into C++ through cppyy, is a separate feature +and is unaffected. numba is therefore still an optional dependency, needed only for that bridge; cffi is not needed at +all any more. + ## I/O ## Core diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdataframe.py b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdataframe.py index e58e85c3ee07a..cadd5a6d5b0ff 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdataframe.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdataframe.py @@ -61,34 +61,40 @@ #### Python code -ROOT also offers the option to compile Python functions with fundamental types and arrays thereof using [Numba](https://numba.pydata.org/). -Such compiled functions can then be used in a C++ expression provided to RDataFrame. +ROOT can translate a Python function over fundamental types and arrays thereof into C++ source, and declare that +to the interpreter. The result is an ordinary C++ function, so it can be used in a C++ expression provided to +RDataFrame and is inlined into the event loop like any other. -The function to be compiled should be decorated with `ROOT.Numba.Declare`, which allows to specify the parameter and -return types. See the following snippet for a simple example or the full tutorial [here](pyroot004__NumbaDeclare_8py.html). +The function to be translated should be decorated with `ROOT.Py.Declare`, which allows to specify the parameter and +return types. See the following snippet for a simple example or the full tutorial [here](df038__NumbaDeclare_8py.html). +The decorator is also available as `ROOT.Numba.Declare`, which declares into the `Numba` C++ namespace instead; that +spelling predates the translation and is kept so that existing code keeps working. ~~~{.py} -@ROOT.Numba.Declare(["float"], "bool") +@ROOT.Py.Declare(["float"], "bool") def myFilter(x): return x > 10 df = ROOT.RDataFrame("myTree", "myFile.root") -sum = df.Filter("Numba::myFilter(x)").Sum("y") +sum = df.Filter("Py::myFilter(x)").Sum("y") print(sum.GetValue()) ~~~ -It also works with collections: `RVec` objects of fundamental types can be transparently converted to/from numpy arrays: +It also works with collections: `RVec`, `std::vector` and `std::array` of fundamental types support the numpy +spelling of element-wise operations and reductions: ~~~{.py} -@ROOT.Numba.Declare(['RVec', 'int'], 'RVec') -def pypowarray(numpyvec, pow): - return numpyvec**pow +@ROOT.Py.Declare(['RVec', 'int'], 'RVec') +def pypowarray(vec, pow): + return vec**pow df.Define('array', 'ROOT::RVecF{1.,2.,3.}')\ - .Define('arraySquared', 'Numba::pypowarray(array, 2)') + .Define('arraySquared', 'Py::pypowarray(array, 2)') ~~~ -Note that this functionality requires the Python packages `numba` and `cffi` to be installed. +The generated C++ is available on the decorated callable as `__cpp_wrapper__`. Only a subset of Python can be +translated; anything outside it is reported when the function is declared, quoting the offending source line. +This functionality requires no third-party Python packages. ### Interoperability with NumPy diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_conversion_maps.py b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_conversion_maps.py index 662add80302ad..ee604b4917f78 100755 --- a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_conversion_maps.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_conversion_maps.py @@ -8,7 +8,7 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ """ -Contains constants needed for _rdf_pyz to convert datatypes for numba declarable types. +Contains constants needed for _rdf_pyz to convert datatypes for declarable types. It is in a separate module so as to avoid a numpy dependency for ROOT. """ try: @@ -22,7 +22,7 @@ float: 'double', } -TREE_TO_NUMBA = { +TREE_TO_CPP = { 'C': 'str', 'Char_t': 'int', 'UChar_t': 'unsigned int', diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py index ab6b98ed92bc2..e0181c50b4a10 100755 --- a/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pythonization/_rdf_pyz.py @@ -74,21 +74,21 @@ def find_type(self, x): import numpy as np except ImportError: raise ImportError("Failed to import numpy during call to determine function signature.") - from ._rdf_conversion_maps import FUNDAMENTAL_PYTHON_TYPES, NUMPY_TO_TREE, TREE_TO_NUMBA + from ._rdf_conversion_maps import FUNDAMENTAL_PYTHON_TYPES, NUMPY_TO_TREE, TREE_TO_CPP if isinstance(x, str): # Can be string constant or can be column name if x in self.col_names: # If x is a column t = self.rdf.GetColumnType(x) - if t in TREE_TO_NUMBA: # The column is a fundamental type from tree - return TREE_TO_NUMBA[t] + if t in TREE_TO_CPP: # The column is a fundamental type from tree + return TREE_TO_CPP[t] match = re.match(r"([\w:]+)<(.+)>", t) if match: container_type, inner_type = match.groups() container_type = container_type.strip() inner_type = inner_type.strip() - inner_mapped = TREE_TO_NUMBA.get(inner_type, inner_type) + inner_mapped = TREE_TO_CPP.get(inner_type, inner_type) return f"{container_type}<{inner_mapped}>" else: return t diff --git a/cmake/scripts/CMakeCPackOptions.cmake.in b/cmake/scripts/CMakeCPackOptions.cmake.in index 3a08ac967cef5..477c6e861a0bc 100644 --- a/cmake/scripts/CMakeCPackOptions.cmake.in +++ b/cmake/scripts/CMakeCPackOptions.cmake.in @@ -16,7 +16,7 @@ if(CPACK_GENERATOR STREQUAL "DEB") set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS_POLICY "=") set(CPACK_DEBIAN_PACKAGE_DEPENDS "g++") set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "python3, python3-numpy") - set(CPACK_DEBIAN_PACKAGE_SUGGESTS "cmake, make, jupyter, python3-numba") + set(CPACK_DEBIAN_PACKAGE_SUGGESTS "cmake, make, jupyter") endif() if(CPACK_GENERATOR STREQUAL "RPM") diff --git a/requirements.txt b/requirements.txt index 01983d5b90c3f..27596ff953453 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,9 +18,10 @@ xgboost # RDataLoader jax -# PyROOT: ROOT.Numba.Declare decorator +# PyROOT: ROOT.NumbaExt, which lets numba-compiled Python call into C++. +# ROOT itself does not use numba; this is here so that the tests of that +# bridge in roottest/python/numba are not silently skipped. numba -cffi # Notebooks: ROOT C++ kernel IPython diff --git a/test/wheels/requirements-ci.txt b/test/wheels/requirements-ci.txt index 707de5430cb0e..9d65aa4ca5b85 100644 --- a/test/wheels/requirements-ci.txt +++ b/test/wheels/requirements-ci.txt @@ -4,10 +4,6 @@ numpy pandas -# PyROOT: ROOT.Numba.Declare decorator -numba -cffi - # Distributed RDataFrame pyspark # Spark backend dask>=2022.08.1 # Dask backend diff --git a/tutorials/CMakeLists.txt b/tutorials/CMakeLists.txt index dfafbb86291ef..502547ac89d08 100644 --- a/tutorials/CMakeLists.txt +++ b/tutorials/CMakeLists.txt @@ -785,6 +785,7 @@ if(pyroot) analysis/dataframe/df017_vecOpsHEP.py analysis/dataframe/df026_AsNumpyArrays.py analysis/dataframe/df032_RDFFromNumpy.py + analysis/dataframe/df038_NumbaDeclare.py hist/hist010_TH1_two_scales.py math/fit/NumericalMinimization.py math/fit/combinedFit.py @@ -816,10 +817,6 @@ if(pyroot) machine_learning/ml_dataloader_Higgs_Classification.py ) - file(GLOB requires_numba RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - analysis/dataframe/df038_NumbaDeclare.py - analysis/dataframe/df101_h1Analysis.py) - file(GLOB requires_pandas RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} analysis/dataframe/df026_AsNumpyArrays.py analysis/dataframe/df035_RDFFromPandas.py @@ -869,7 +866,6 @@ if(pyroot) set(fixtureLists requires_keras - requires_numba requires_numpy requires_pandas requires_sklearn diff --git a/tutorials/analysis/dataframe/df038_NumbaDeclare.py b/tutorials/analysis/dataframe/df038_NumbaDeclare.py index f62e67e41a1ca..b14b5b5f56e92 100644 --- a/tutorials/analysis/dataframe/df038_NumbaDeclare.py +++ b/tutorials/analysis/dataframe/df038_NumbaDeclare.py @@ -2,9 +2,11 @@ ## \ingroup tutorial_dataframe ## \notebook -nodraw ## This tutorial illustrates how PyROOT supports declaring C++ callables from -## Python callables making them, for example, usable with RDataFrame. The feature -## uses the numba Python package for just-in-time compilation of the Python callable -## and supports fundamental types and ROOT::RVec thereof. +## Python callables making them, for example, usable with RDataFrame. The Python +## function is translated to C++ source and declared to the interpreter, so the +## result is an ordinary C++ function that is inlined into the event loop. +## Fundamental types are supported, along with ROOT::RVec, std::vector and +## std::array of them, and any C++ class the interpreter already knows. ## ## \macro_code ## \macro_output @@ -16,11 +18,15 @@ # To mark a Python callable to be used from C++, you have to use the decorator # provided by PyROOT passing the C++ types of the input arguments and the return -# value. +# value. The generated C++ is available afterwards as pypow.__cpp_wrapper__. @ROOT.Numba.Declare(['float', 'int'], 'float') def pypow(x, y): return x**y +# The decorator is also available as ROOT.Py.Declare, which is the same thing +# declared into the Py C++ namespace; ROOT.Numba.Declare predates the C++ +# translation and is kept so that existing code keeps working. + # The Python callable is now available from C++ in the Numba namespace. # For example, we can use it from the interpreter. ROOT.gInterpreter.ProcessLine('cout << "2^3 = " << Numba::pypow(2, 3) << endl;') @@ -32,10 +38,9 @@ def pypow(x, y): print('pypow({}, 3) = {}'.format(data['x'], data['x_pow3'])) -# ROOT uses the numba Python package to create C++ functions from python ones. -# We support as input and return types of the callable fundamental types and -# ROOT::RVec thereof. See the following callable computing the power of the -# elements in an array. +# The supported input and return types are the fundamental types and the +# containers of them: ROOT::RVec, std::vector and std::array. See the following +# callable computing the power of the elements in an array. @ROOT.Numba.Declare(['RVecF', 'int'], 'RVecF') def pypowarray(x, y): return x**y diff --git a/tutorials/analysis/dataframe/df101_h1Analysis.py b/tutorials/analysis/dataframe/df101_h1Analysis.py index 96cb4ad8509a3..b0a185b4437f0 100644 --- a/tutorials/analysis/dataframe/df101_h1Analysis.py +++ b/tutorials/analysis/dataframe/df101_h1Analysis.py @@ -3,7 +3,7 @@ ROOT.EnableImplicitMT(4) -# Declare filters on RVec objects and JIT with Numba +# Declare filters on RVec objects, translated to C++ and jitted with cling @ROOT.Numba.Declare(["int", "int", "RVecI"], "bool") def ik_ipi_nhitrp_cut(ik, ipi, nhitrp): return nhitrp[ik - 1] * nhitrp[ipi - 1] > 1 From 5bb0ce57bbe0ca05af831a82e3aabe6bfb6b6856 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 20 Nov 2026 21:26:29 +0100 Subject: [PATCH 7/8] [Python] Skip the pydeclare tests that throw C++ exceptions on Mac ARM The PyDeclareRaisesInsteadOfMisbehaving tests check that the translated code throws where Python would raise, and that the exception reaches Python. On macOS arm64, C++ exceptions thrown from JIT-compiled code can't be caught reliably: depending on what was JIT-compiled before in the process, the unwinder doesn't find the frames and the process aborts with "terminating due to uncaught exception". This aborted the whole pydeclare test on all macOS arm64 CI runners. Skip that test class on macOS arm64, as the cppjit tests and the distrdf backend tests already do for the same limitation. Assisted-by: claude-code:claude-opus-5-5 --- bindings/pyroot/pythonizations/test/pydeclare.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bindings/pyroot/pythonizations/test/pydeclare.py b/bindings/pyroot/pythonizations/test/pydeclare.py index be4f4f0917732..f3f487f34572f 100644 --- a/bindings/pyroot/pythonizations/test/pydeclare.py +++ b/bindings/pyroot/pythonizations/test/pydeclare.py @@ -14,6 +14,7 @@ import gc import importlib.util import os +import platform import shutil import sys import tempfile @@ -26,6 +27,11 @@ default_test_inputs = [-1.0, 0.0, 100.0] +# C++ exceptions thrown from JIT-compiled code can't be caught reliably on +# macOS arm64: depending on what was JIT-compiled before, the unwinder doesn't +# find the frames and the process aborts. +IS_MAC_ARM = platform.system() == "Darwin" and platform.machine() == "arm64" + class NumbaDeclareSimple(unittest.TestCase): """ @@ -1440,6 +1446,7 @@ def myabs(x): self.check("mix_abs", myabs, ["unsigned int"], "unsigned int", (3,), 3) +@unittest.skipIf(IS_MAC_ARM, "JIT exceptions can not be caught reliably on Mac ARM") class PyDeclareRaisesInsteadOfMisbehaving(unittest.TestCase): """Where Python raises, the translation must raise too. From 833fe723e9f52dda658147ed19045f8eeb7488cc Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 20 Nov 2026 21:28:50 +0100 Subject: [PATCH 8/8] [Python] Fix ruff complaints about the Python-to-C++ transpiler code Format the new files of the transpiler and its test with `ruff format --preview`, which is what the code analysis CI checks, and add the blank line after the imports in df038_NumbaDeclare.py that `ruff check` wants (I001). Assisted-by: claude-code:claude-opus-5-5 --- .../python/ROOT/_pydeclare/ast_backend.py | 48 ++++++++------ .../python/ROOT/_pydeclare/cpptypes.py | 11 +++- .../python/ROOT/_pydeclare/emit.py | 62 +++++++++---------- .../pyroot/pythonizations/test/pydeclare.py | 54 ++++++---------- .../analysis/dataframe/df038_NumbaDeclare.py | 1 + 5 files changed, 87 insertions(+), 89 deletions(-) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py index f4c19cc106a4d..18c2695df7b98 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/ast_backend.py @@ -45,9 +45,21 @@ BUILTIN_CASTS = {"float", "int", "bool"} #: Builtins that are safe to run at declaration time when folding constants. -PURE_BUILTINS = frozenset( - ["abs", "bool", "float", "int", "len", "max", "min", "pow", "round", "sum", "range", "list", "tuple"] -) +PURE_BUILTINS = frozenset([ + "abs", + "bool", + "float", + "int", + "len", + "max", + "min", + "pow", + "round", + "sum", + "range", + "list", + "tuple", +]) class _ModuleRef: @@ -175,7 +187,7 @@ def translate(self): "The variable '{}' is only assigned inside a nested block but is read outside it. " "If the block does not run, Python raises UnboundLocalError, which cannot be " "reproduced in C++.\n" - " Assign it before the block, for example \"{} = 0.0\".".format(name, name) + ' Assign it before the block, for example "{} = 0.0".'.format(name, name) ) # Pass 2: the real one. @@ -1135,21 +1147,19 @@ def _try_fold(self, node): _IMPURE_NUMPY_MODULES = ("numpy.random", "numpy.random.mtrand", "numpy.testing") #: numpy functions that touch the filesystem. -_IMPURE_NUMPY_FUNCTIONS = frozenset( - [ - "load", - "loadtxt", - "save", - "savetxt", - "savez", - "savez_compressed", - "fromfile", - "tofile", - "genfromtxt", - "memmap", - "fromregex", - ] -) +_IMPURE_NUMPY_FUNCTIONS = frozenset([ + "load", + "loadtxt", + "save", + "savetxt", + "savez", + "savez_compressed", + "fromfile", + "tofile", + "genfromtxt", + "memmap", + "fromregex", +]) class _BoundMethod: diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py index eeb603eda06c4..26b96deac1336 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/cpptypes.py @@ -158,9 +158,14 @@ def __hash__(self): INTEGRAL_TYPES = frozenset(INTEGRAL_RANKS) FLOATING_TYPES = frozenset(FLOATING_RANKS) -UNSIGNED_TYPES = frozenset( - ["bool", "unsigned char", "unsigned short", "unsigned int", "unsigned long", "unsigned long long"] -) +UNSIGNED_TYPES = frozenset([ + "bool", + "unsigned char", + "unsigned short", + "unsigned int", + "unsigned long", + "unsigned long long", +]) # Spellings the user may write, normalised to the canonical form above. _FUND_ALIASES = { diff --git a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py index 0c712559c5ffc..81c24d6f28b41 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_pydeclare/emit.py @@ -504,38 +504,36 @@ def slice_(obj, start, stop, step, fail): } #: Functions whose result is always a double, whatever the argument type. -_DOUBLE_RESULT = frozenset( - [ - "Sqrt", - "Cbrt", - "Exp", - "Exp2", - "Expm1", - "Log", - "Log2", - "Log10", - "Log1p", - "Sin", - "Cos", - "Tan", - "Asin", - "Acos", - "Atan", - "Sinh", - "Cosh", - "Tanh", - "Asinh", - "Acosh", - "Atanh", - "Erf", - "Erfc", - "Lgamma", - "Tgamma", - "Atan2", - "Hypot", - "Fmod", - ] -) +_DOUBLE_RESULT = frozenset([ + "Sqrt", + "Cbrt", + "Exp", + "Exp2", + "Expm1", + "Log", + "Log2", + "Log10", + "Log1p", + "Sin", + "Cos", + "Tan", + "Asin", + "Acos", + "Atan", + "Sinh", + "Cosh", + "Tanh", + "Asinh", + "Acosh", + "Atanh", + "Erf", + "Erfc", + "Lgamma", + "Tgamma", + "Atan2", + "Hypot", + "Fmod", +]) BINARY_FUNCTIONS = { "arctan2": "Atan2", diff --git a/bindings/pyroot/pythonizations/test/pydeclare.py b/bindings/pyroot/pythonizations/test/pydeclare.py index f3f487f34572f..8ba2504f6eef0 100644 --- a/bindings/pyroot/pythonizations/test/pydeclare.py +++ b/bindings/pyroot/pythonizations/test/pydeclare.py @@ -1031,12 +1031,8 @@ def test_reading_a_file_at_declaration_time(self): ) def test_sets_and_tuples(self): - self.assertRefused( - "def f(x):\n s = {1, 2}\n return x\n", ["double"], "double", "Set literals" - ) - self.assertRefused( - "def f(x):\n t = (x, x)\n return t[0]\n", ["double"], "double", "Tuple literals" - ) + self.assertRefused("def f(x):\n s = {1, 2}\n return x\n", ["double"], "double", "Set literals") + self.assertRefused("def f(x):\n t = (x, x)\n return t[0]\n", ["double"], "double", "Tuple literals") def test_lambda_in_the_body(self): self.assertRefused( @@ -1053,9 +1049,7 @@ def test_with_and_global(self): "double", "'With' is not supported", ) - self.assertRefused( - "def f(x):\n global g\n return x\n", ["double"], "double", "'Global' is not supported" - ) + self.assertRefused("def f(x):\n global g\n return x\n", ["double"], "double", "'Global' is not supported") def test_f_strings_and_generators(self): self.assertRefused( @@ -1066,19 +1060,13 @@ def test_f_strings_and_generators(self): ) def test_star_args(self): - self.assertRefused( - "def f(*args):\n return args[0]\n", ["double"], "double", "*args" - ) + self.assertRefused("def f(*args):\n return args[0]\n", ["double"], "double", "*args") def test_multi_dimensional_indexing(self): - self.assertRefused( - "def f(v):\n return v[0, 1]\n", ["RVecD"], "double", "Multi-dimensional indexing" - ) + self.assertRefused("def f(v):\n return v[0, 1]\n", ["RVecD"], "double", "Multi-dimensional indexing") def test_signature_mismatch(self): - self.assertRefused( - "def f(x, y):\n return x + y\n", ["double"], "double", "takes 2 argument" - ) + self.assertRefused("def f(x, y):\n return x + y\n", ["double"], "double", "takes 2 argument") def test_mutating_an_array_that_python_would_alias(self): """'w = v' aliases in Python and copies here, so writing through it @@ -1239,20 +1227,18 @@ def total(v): def test_slices(self): cases = [(np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]),)] - for i, body in enumerate( - [ - "v[1:4]", - "v[-2:]", - "v[:3]", - "v[::2]", - "v[1::2]", - "v[10:20]", - "v[4:1:-1]", - "v[::-1]", - "v[-100:100]", - "v[3:1]", - ] - ): + for i, body in enumerate([ + "v[1:4]", + "v[-2:]", + "v[:3]", + "v[::2]", + "v[1::2]", + "v[10:20]", + "v[4:1:-1]", + "v[::-1]", + "v[-100:100]", + "v[3:1]", + ]): namespace = {"np": np} exec("def sliced(v):\n return {}\n".format(body), namespace) # noqa: S102 func = namespace["sliced"] @@ -1534,9 +1520,7 @@ def total(v): name = "spelling_{}".format(i) ROOT.Py.Declare([spelling], "double", name=name)(total) - self.assertAlmostEqual( - getattr(ROOT.Py, name)(ROOT.VecOps.RVec("float")([1.0, 2.0, 3.0])), 6.0 - ) + self.assertAlmostEqual(getattr(ROOT.Py, name)(ROOT.VecOps.RVec("float")([1.0, 2.0, 3.0])), 6.0) def _as_cpp(value): diff --git a/tutorials/analysis/dataframe/df038_NumbaDeclare.py b/tutorials/analysis/dataframe/df038_NumbaDeclare.py index b14b5b5f56e92..a6b07df6527c6 100644 --- a/tutorials/analysis/dataframe/df038_NumbaDeclare.py +++ b/tutorials/analysis/dataframe/df038_NumbaDeclare.py @@ -16,6 +16,7 @@ import ROOT + # To mark a Python callable to be used from C++, you have to use the decorator # provided by PyROOT passing the C++ types of the input arguments and the return # value. The generated C++ is available afterwards as pypow.__cpp_wrapper__.