diff --git a/packages/bigframes/bigframes/extensions/bigframes/series_accessor.py b/packages/bigframes/bigframes/extensions/bigframes/series_accessor.py index 8379e6a145a0..c9026595d97e 100644 --- a/packages/bigframes/bigframes/extensions/bigframes/series_accessor.py +++ b/packages/bigframes/bigframes/extensions/bigframes/series_accessor.py @@ -44,17 +44,17 @@ def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: def _to_series(self, bf_series: series.Series) -> S: return cast(S, bf_series) - @property - def ai(self) -> BigframesAiSeriesAccessor[T, S]: - return BigframesAiSeriesAccessor(self._obj) - @property def aead(self) -> BigframesAeadSeriesAccessor[T, S]: return BigframesAeadSeriesAccessor(self._obj) + @property + def ai(self) -> BigframesAiSeriesAccessor[T, S]: + return BigframesAiSeriesAccessor(self._obj) + @log_adapter.class_logger -class BigframesAiSeriesAccessor(core_accessor.AiSeriesAccessor[T, S]): +class BigframesAeadSeriesAccessor(core_accessor.AeadSeriesAccessor[T, S]): def __init__(self, bf_obj: S): super().__init__(bf_obj) @@ -71,7 +71,7 @@ def _to_series(self, bf_series: series.Series) -> S: @log_adapter.class_logger -class BigframesAeadSeriesAccessor(core_accessor.AeadSeriesAccessor[T, S]): +class BigframesAiSeriesAccessor(core_accessor.AiSeriesAccessor[T, S]): def __init__(self, bf_obj: S): super().__init__(bf_obj) diff --git a/packages/bigframes/bigframes/extensions/core/series_accessor.py b/packages/bigframes/bigframes/extensions/core/series_accessor.py index 440e6731aba2..79a9e511c3c6 100644 --- a/packages/bigframes/bigframes/extensions/core/series_accessor.py +++ b/packages/bigframes/bigframes/extensions/core/series_accessor.py @@ -44,13 +44,13 @@ class BigQuerySeriesAccessor( @property @abc.abstractmethod - def ai(self) -> AiSeriesAccessor[T, S]: - """Accessor for BigQuery ai functions.""" + def aead(self) -> AeadSeriesAccessor[T, S]: + """Accessor for BigQuery aead functions.""" @property @abc.abstractmethod - def aead(self) -> AeadSeriesAccessor[T, S]: - """Accessor for BigQuery aead functions.""" + def ai(self) -> AiSeriesAccessor[T, S]: + """Accessor for BigQuery ai functions.""" def deterministic_decrypt_bytes( self, @@ -1112,10 +1112,6 @@ def unix_date( return self._to_series(cast(series.Series, result)) -class AiSeriesAccessor(series_tvf_mixins.AITVFMixin[T, S]): - """Series accessor for BigQuery ai functions.""" - - class AeadSeriesAccessor(abstract_series_accessor.AbstractBigQuerySeriesAccessor[T, S]): """Series accessor for BigQuery aead functions.""" @@ -1227,3 +1223,7 @@ def encrypt( additional_data, ) return self._to_series(cast(series.Series, result)) + + +class AiSeriesAccessor(series_tvf_mixins.AITVFMixin[T, S]): + """Series accessor for BigQuery ai functions.""" diff --git a/packages/bigframes/bigframes/extensions/pandas/series_accessor.py b/packages/bigframes/bigframes/extensions/pandas/series_accessor.py index 204f1e0d2cf3..9c33996c4219 100644 --- a/packages/bigframes/bigframes/extensions/pandas/series_accessor.py +++ b/packages/bigframes/bigframes/extensions/pandas/series_accessor.py @@ -51,17 +51,17 @@ def _to_dataframe(self, bf_df: dataframe.DataFrame) -> T: def _to_series(self, bf_series: series.Series) -> S: return cast(S, bf_series.to_pandas(ordered=True)) - @property - def ai(self) -> PandasAiSeriesAccessor[T, S]: - return PandasAiSeriesAccessor(self._obj) - @property def aead(self) -> PandasAeadSeriesAccessor[T, S]: return PandasAeadSeriesAccessor(self._obj) + @property + def ai(self) -> PandasAiSeriesAccessor[T, S]: + return PandasAiSeriesAccessor(self._obj) + @log_adapter.class_logger -class PandasAiSeriesAccessor(core_accessor.AiSeriesAccessor[T, S]): +class PandasAeadSeriesAccessor(core_accessor.AeadSeriesAccessor[T, S]): def __init__(self, pandas_obj: S): super().__init__(pandas_obj) @@ -80,7 +80,7 @@ def _to_series(self, bf_series: series.Series) -> S: @log_adapter.class_logger -class PandasAeadSeriesAccessor(core_accessor.AeadSeriesAccessor[T, S]): +class PandasAiSeriesAccessor(core_accessor.AiSeriesAccessor[T, S]): def __init__(self, pandas_obj: S): super().__init__(pandas_obj) diff --git a/packages/bigframes/scripts/bigquery_generator/__init__.py b/packages/bigframes/scripts/bigquery_generator/__init__.py new file mode 100644 index 000000000000..58d482ea3866 --- /dev/null +++ b/packages/bigframes/scripts/bigquery_generator/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/packages/bigframes/scripts/bigquery_generator/constants.py b/packages/bigframes/scripts/bigquery_generator/constants.py new file mode 100644 index 000000000000..9a9a35339b25 --- /dev/null +++ b/packages/bigframes/scripts/bigquery_generator/constants.py @@ -0,0 +1,194 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib + +import jinja2 + +SCRIPTS_DIRECTORY = pathlib.Path(__file__).parent.parent.absolute() +PACKAGE_ROOT = SCRIPTS_DIRECTORY.parent +CODE_ROOT = PACKAGE_ROOT / "bigframes" +SCRIPT_PATH_RELATIVE = ( + pathlib.Path(__file__).relative_to(PACKAGE_ROOT).parent.parent + / "generate_bigframes_bigquery.py" +) + + +# Directory containing the YAML files +DATA_DIR = SCRIPTS_DIRECTORY / "data" / "sql-functions" +# Directory where the generated Python files will be placed +OUTPUT_DIR = CODE_ROOT / "operations" / "googlesql" +# Directory where the generated test files will be placed +TEST_OUTPUT_DIR = PACKAGE_ROOT / "tests" / "unit" / "bigquery" / "generated" +# Directory containing the Jinja2 templates +TEMPLATE_DIR = SCRIPTS_DIRECTORY / "templates" + +PYTHON_BUILTINS = { + "abs", + "all", + "any", + "ascii", + "bin", + "bool", + "breakpoint", + "bytearray", + "bytes", + "callable", + "chr", + "classmethod", + "compile", + "complex", + "delattr", + "dict", + "dir", + "divmod", + "enumerate", + "eval", + "exec", + "filter", + "float", + "format", + "frozenset", + "getattr", + "globals", + "hasattr", + "hash", + "help", + "hex", + "id", + "input", + "int", + "isinstance", + "issubclass", + "iter", + "len", + "list", + "locals", + "map", + "max", + "memoryview", + "min", + "next", + "object", + "oct", + "open", + "ord", + "pow", + "print", + "property", + "range", + "repr", + "reversed", + "round", + "set", + "setattr", + "slice", + "sorted", + "staticmethod", + "str", + "sum", + "super", + "tuple", + "type", + "vars", + "zip", +} + +DTYPE_MAP = { + "binary": "dtypes.BYTES_DTYPE", + "string": "dtypes.STRING_DTYPE", + "int64": "dtypes.INT_DTYPE", + "i64": "dtypes.INT_DTYPE", + "float64": "dtypes.FLOAT_DTYPE", + "fp64": "dtypes.FLOAT_DTYPE", + "bool": "dtypes.BOOL_DTYPE", + "boolean": "dtypes.BOOL_DTYPE", + "geography": "dtypes.GEO_DTYPE", + "json": "dtypes.JSON_DTYPE", + "date": "dtypes.DATE_DTYPE", + "time": "dtypes.TIME_DTYPE", + "datetime": "dtypes.DATETIME_DTYPE", + "timestamp": "dtypes.TIMESTAMP_DTYPE", + "decimal<38,9>": "dtypes.NUMERIC_DTYPE", + "decimal<76,38>": "dtypes.BIGNUMERIC_DTYPE", +} + +PY_TYPE_MAP = { + "binary": "bytes", + "string": "str", + "int64": "int", + "i64": "int", + "float64": "float", + "fp64": "float", + "bool": "bool", + "boolean": "bool", + "geography": "Any", + "json": "Any", + "date": "datetime.date", + "time": "datetime.time", + "datetime": "datetime.datetime", + "timestamp": "datetime.datetime", + "struct": "dict", + "decimal<38,9>": "decimal.Decimal", + "decimal<76,38>": "decimal.Decimal", + "interval_day": "datetime.timedelta", +} + +RUFF_COMMON_ARGS = [ + "--target-version=py310", + "--line-length=88", +] +RUFF_CHECK_ARGS = [ + "check", + "--select", + "I,F", + "--fix", +] + RUFF_COMMON_ARGS +RUFF_FORMAT_ARGS = [ + "format", +] + RUFF_COMMON_ARGS + + +def _load_templates() -> dict[str, jinja2.Template]: + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(TEMPLATE_DIR), + trim_blocks=True, + lstrip_blocks=True, + ) + return { + "operation": env.get_template("operation.py.j2"), + "test_operation": env.get_template("test_operation.py.j2"), + "license": env.get_template("license.py.j2"), + "signature_def": env.get_template("signature_def.py.j2"), + "core_series_accessor": env.get_template("core_series_accessor.py.j2"), + "bigframes_series_accessor": env.get_template( + "bigframes_series_accessor.py.j2" + ), + "pandas_series_accessor": env.get_template("pandas_series_accessor.py.j2"), + } + + +TEMPLATES: dict[str, jinja2.Template] = _load_templates() + + +def __getattr__(name: str): + global _templates + + if name != "TEMPLATES": + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") + + if _templates is None: + _templates = _load_templates() + + return _templates diff --git a/packages/bigframes/scripts/bigquery_generator/data_models.py b/packages/bigframes/scripts/bigquery_generator/data_models.py new file mode 100644 index 000000000000..72b02bd026c6 --- /dev/null +++ b/packages/bigframes/scripts/bigquery_generator/data_models.py @@ -0,0 +1,194 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Data models for BigQuery code generator. + +`BQ*` models the Substrait YAML extension structure of BigQuery SQL functions, +while `BigFrames*` models the Jinja template outputs. + +BQ* Class Relations: +==================== ++-------------------+ +| BQModule | ++-------------------+ + | + | functions: list[BQFunc] + v ++-------------------+ +| BQFunc | ++-------------------+ + | + | impls: list[BQFuncImpl] + v ++-------------------+ +| BQFuncImpl | ++-------------------+ + | + | args: list[BQFuncArg] + v ++-------------------+ +| BQFuncArg | ++-------------------+ + +BigFrames* Class Relations: +================================= + +--------------------------------+ + | Accessor |<---+ children: list[Accessor] + +--------------------------------+----+ (nested namespace hierarchy) + | + | functions: list[BigFramesFunc] + v + +--------------------------------+ + | BigFramesFunc | + +--------------------------------+ + | + | args: list[BigFramesFuncArg] + v + +--------------------------------+ + | BigFramesFuncArg | + +--------------------------------+ + + ---------------------------------- + + +--------------------------------+ + | BigFramesOp | (Standalone data model for op defs) + +--------------------------------+ +""" + +from __future__ import annotations + +import dataclasses +import pathlib +from typing import Any + +from . import constants + + +@dataclasses.dataclass +class BQFuncArg: + name: str + value: str # The type of the arg + optional: bool + keyword_only: bool + + +@dataclasses.dataclass +class BQFuncImpl: + args: list[BQFuncArg] + return_type: str + + @property + def uses_any1(self) -> bool: + if "any1" in self.return_type: + return True + + return any("any1" in arg.value for arg in self.args) + + def to_dict(self) -> dict[str, Any]: + result = dataclasses.asdict(self) + + result["uses_any1"] = self.uses_any1 + + # We cannot use "return" as a field name, but we need to use it + # as a key for template rendering + result["return"] = self.return_type + del result["return_type"] + + return result + + +@dataclasses.dataclass +class BQFunc: + name: str + op_base_name: str + description: str + impls: list[BQFuncImpl] + series_accessor_arg: str | None + + +@dataclasses.dataclass +class BQModule: + yaml_file: pathlib.Path + functions: list[BQFunc] + + @property + def module_path(self) -> pathlib.Path: + return self.yaml_file.relative_to(constants.DATA_DIR).with_suffix("") + + @property + def namespace(self) -> tuple[str, ...]: + parts = self.module_path.parts + if "global_namespace" in parts: + return tuple() + return parts + + @property + def is_global(self) -> bool: + return "global_namespace" in self.module_path.parts + + +@dataclasses.dataclass +class BigFramesOp: + internal_name: str + sql_name: str + arg_specs: str + signature: str + signature_definition: str | None + + +@dataclasses.dataclass +class BigFramesFuncArg: + name: str + types: set[str] + optional: bool + keyword_only: bool + + @property + def type_hint(self) -> str: + types = [constants.PY_TYPE_MAP.get(t, "Any") for t in sorted(self.types)] + [ + "Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]" + ] + + if len(types) > 1: + return "Union[" + ", ".join(sorted(set(types))) + "]" + + return types[0] + + @property + def default(self) -> str | None: + if self.optional: + return "sentinels.Sentinel.ARGUMENT_DEFAULT" + return None + + +@dataclasses.dataclass +class BigFramesFunc: + name: str + op_name: str + description: str + args: list[BigFramesFuncArg] + series_accessor_arg: str | None + import_module: str | None = None + + +@dataclasses.dataclass +class Accessor: + class_name: str + bigframes_class_name: str + pandas_class_name: str + is_root: bool + description: str + children: list[Accessor] + functions: list[BigFramesFunc] + prop_name: str | None = None diff --git a/packages/bigframes/scripts/bigquery_generator/file_generator.py b/packages/bigframes/scripts/bigquery_generator/file_generator.py new file mode 100644 index 000000000000..14df5ff98515 --- /dev/null +++ b/packages/bigframes/scripts/bigquery_generator/file_generator.py @@ -0,0 +1,115 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import subprocess +import sys + +from . import constants, data_models, template_renderer + + +def _ensure_init_py(directory: pathlib.Path, limit_dir: pathlib.Path): + """Ensures __init__.py exists in the directory and its parents up to limit_dir.""" + curr = directory + while curr != limit_dir and curr != curr.parent: + init_file = curr / "__init__.py" + if not init_file.exists(): + print(f" Creating {init_file}") + content = constants.TEMPLATES["license"].render() + with open(init_file, "w", encoding="utf-8") as f: + f.write(content) + curr = curr.parent + + +def _write_file(content: str, output_file: pathlib.Path, limit_dir: pathlib.Path): + output_file.parent.mkdir(parents=True, exist_ok=True) + _ensure_init_py(output_file.parent, limit_dir) + + with open(output_file, "w", encoding="utf-8") as f: + f.write(content) + print(f" Generated {output_file}") + + +def _run_ruff(): + targets = [ + constants.OUTPUT_DIR, + constants.TEST_OUTPUT_DIR, + constants.CODE_ROOT / "extensions", + ] + + subprocess.run( + [sys.executable, "-m", "ruff"] + constants.RUFF_CHECK_ARGS + targets, + check=True, + ) + + subprocess.run( + [sys.executable, "-m", "ruff"] + constants.RUFF_FORMAT_ARGS + targets, + check=True, + ) + + +def _generate_op_defs(bq_module: data_models.BQModule): + content = template_renderer.render_operation(bq_module) + if not content: + return + + output_file = constants.OUTPUT_DIR.joinpath(bq_module.module_path).with_suffix( + ".py" + ) + + _write_file(content, output_file, constants.OUTPUT_DIR.parent) + + +def _generate_tests(bq_module: data_models.BQModule): + content = template_renderer.render_tests(bq_module) + if not content: + return + + output_file = constants.TEST_OUTPUT_DIR.joinpath( + bq_module.module_path.with_name(f"test_{bq_module.module_path.name}") + ).with_suffix(".py") + + _write_file(content, output_file, constants.TEST_OUTPUT_DIR.parent) + + +def _generate_accesor(bq_modules: list[data_models.BQModule]): + (core_content, pd_content, bf_content) = template_renderer.render_accessor( + bq_modules + ) + + core_output_file = ( + constants.CODE_ROOT / "extensions" / "core" / "series_accessor.py" + ) + _write_file(core_content, core_output_file, constants.CODE_ROOT) + + pd_output_file = ( + constants.CODE_ROOT / "extensions" / "pandas" / "series_accessor.py" + ) + _write_file(pd_content, pd_output_file, constants.CODE_ROOT) + + bf_output_file = ( + constants.CODE_ROOT / "extensions" / "bigframes" / "series_accessor.py" + ) + _write_file(bf_content, bf_output_file, constants.CODE_ROOT) + + +def generate(bq_modules: list[data_models.BQModule]): + for bq_module in bq_modules: + _generate_op_defs(bq_module) + _generate_tests(bq_module) + + _generate_accesor(bq_modules) + + # Ruff format + _run_ruff() diff --git a/packages/bigframes/scripts/bigquery_generator/template_renderer.py b/packages/bigframes/scripts/bigquery_generator/template_renderer.py new file mode 100644 index 000000000000..ee9944e79a13 --- /dev/null +++ b/packages/bigframes/scripts/bigquery_generator/template_renderer.py @@ -0,0 +1,301 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Renders jinja template with module data parsed from yaml. +""" + +from . import constants, data_models + + +def _get_concrete_type_expr(yaml_type: str) -> str: + if yaml_type in constants.DTYPE_MAP: + return constants.DTYPE_MAP[yaml_type] + if yaml_type.startswith("list<") and yaml_type.endswith(">"): + inner = yaml_type[5:-1] + if inner in constants.DTYPE_MAP: + return f"dtypes.list_type({constants.DTYPE_MAP[inner]})" + raise ValueError(f"Not a concrete type: {yaml_type}") + + +def _is_concrete_type(yaml_type: str) -> bool: + try: + _get_concrete_type_expr(yaml_type) + return True + except ValueError: + return False + + +def _validate_types(impls: list[data_models.BQFuncImpl]) -> None: + for impl in impls: + for arg in impl.args: + val = arg.value + if val == "any1": + continue + if val.startswith("list<") and val.endswith(">"): + inner = val[5:-1] + if inner != "any1" and inner not in constants.DTYPE_MAP: + raise ValueError(f"Unsupported inner type: {inner}") + continue + if val == "struct": + continue + if val not in constants.DTYPE_MAP: + raise ValueError(f"Unsupported type: {val}") + + ret = impl.return_type + if ret == "any1": + continue + if ret.startswith("list<") and ret.endswith(">"): + inner = ret[5:-1] + if inner != "any1" and inner not in constants.DTYPE_MAP: + raise ValueError(f"Unsupported inner type: {inner}") + continue + if ret not in constants.DTYPE_MAP: + raise ValueError(f"Unsupported type: {ret}") + + +def render_signature_def( + bq_func: data_models.BQFunc, +) -> tuple[str, str | None]: + """ + Returns the signature function name and it's definition. + If the signature function can be inlined, the first return value is the lambda, + and the second value is None. + """ + return_types = {impl.return_type for impl in bq_func.impls} + # Optimization: if all impls return the same concrete type, + # inline the signature function as a lambda + if len(return_types) == 1: + ret_type = next(iter(return_types)) + if _is_concrete_type(ret_type): + sig_expr = f"lambda *args: {_get_concrete_type_expr(ret_type)}" + return sig_expr, None + + _validate_types(bq_func.impls) + + sig_func_name = f"_{bq_func.op_base_name.upper()}_SIG" + + max_args = max(len(impl.args) for impl in bq_func.impls) + + rendered = constants.TEMPLATES["signature_def"].render( + func_name=sig_func_name, + max_args=max_args, + impls=[impl.to_dict() for impl in bq_func.impls], + sql_name=bq_func.name, + dtype_map=constants.DTYPE_MAP, + ) + + return sig_func_name, rendered + + +def _get_bigframes_func_args( + bq_func: data_models.BQFunc, +) -> list[data_models.BigFramesFuncArg]: + """ + Coalesces arguments from all the implementations of this function, + and return them in the order of appearance in the yaml file + """ + args_by_name = {} + arg_order = [] + arg_appearances = {} + for impl in bq_func.impls: + seen_in_impl = set() + for bq_func_arg in impl.args: + name = bq_func_arg.name + seen_in_impl.add(name) + if name not in args_by_name: + args_by_name[name] = data_models.BigFramesFuncArg( + name=name, + types=set(), + optional=bq_func_arg.optional, + keyword_only=bq_func_arg.keyword_only, + ) + arg_order.append(name) + else: + # If it was marked optional or keyword_only in any previous impl, keep it. + # Or if this impl marks it as optional/keyword_only, update it. + if bq_func_arg.optional: + args_by_name[name].optional = True + if bq_func_arg.keyword_only: + args_by_name[name].keyword_only = True + args_by_name[name].types.add(bq_func_arg.value) + for name in seen_in_impl: + arg_appearances[name] = arg_appearances.get(name, 0) + 1 + + # If an argument is not in all impls, it must be optional overall + num_impls = len(bq_func.impls) + for name, count in arg_appearances.items(): + if count < num_impls: + args_by_name[name].optional = True + + return [args_by_name[name] for name in arg_order] + + +def _to_bigframes_op(bq_func: data_models.BQFunc) -> data_models.BigFramesOp: + arg_specs = [] + for bf_func_arg in _get_bigframes_func_args(bq_func): + spec = "googlesql.ArgSpec(" + if bf_func_arg.keyword_only: + spec += f'arg_name="{bf_func_arg.name}", ' + if bf_func_arg.optional: + spec += "optional=True, " + spec = spec.rstrip(", ") + ")" + arg_specs.append(spec) + + arg_specs_str = ", ".join(arg_specs) + if len(arg_specs) == 1: + arg_specs_str += "," + + (signature, signature_definition) = render_signature_def(bq_func) + + return data_models.BigFramesOp( + internal_name=f"_{bq_func.op_base_name.upper()}_OP", + sql_name=bq_func.name.upper(), + arg_specs=arg_specs_str, + signature=signature, + signature_definition=signature_definition, + ) + + +def _to_bigframes_func(bq_func: data_models.BQFunc) -> data_models.BigFramesFunc: + python_name = bq_func.op_base_name + if python_name in constants.PYTHON_BUILTINS: + python_name = python_name + "_" + + return data_models.BigFramesFunc( + name=python_name, + op_name=f"_{bq_func.op_base_name.upper()}_OP", + description=bq_func.description, + args=_get_bigframes_func_args(bq_func), + series_accessor_arg=bq_func.series_accessor_arg, + ) + + +def render_operation( + bq_module: data_models.BQModule, +) -> str: + if not bq_module.functions: + # If there are no function definitions, do not generate an empty file. + return "" + + ops = [] + functions = [] + + for bq_func in bq_module.functions: + ops.append(_to_bigframes_op(bq_func)) + functions.append(_to_bigframes_func(bq_func)) + + return constants.TEMPLATES["operation"].render( + yaml_path=bq_module.yaml_file.relative_to(constants.PACKAGE_ROOT), + script_path=constants.SCRIPT_PATH_RELATIVE, + ops=ops, + functions=functions, + ) + + +def render_tests(bq_module: data_models.BQModule) -> str: + if not bq_module.functions: + # If there are no function definitions, do not generate an empty test file. + return "" + + import_path = "bigframes.operations.googlesql." + ".".join( + bq_module.module_path.parts + ) + functions = [] + for bq_func in bq_module.functions: + functions.append(_to_bigframes_func(bq_func)) + + return constants.TEMPLATES["test_operation"].render( + yaml_path=bq_module.yaml_file.relative_to(constants.PACKAGE_ROOT), + script_path=constants.SCRIPT_PATH_RELATIVE, + import_path=import_path, + short_name=bq_module.module_path.name, + is_global=bq_module.is_global, + functions=functions, + ) + + +def _create_accessor_class_name(namespace: tuple[str, ...], prefix: str = "") -> str: + if not namespace: + return f"{prefix}BigQuerySeriesAccessor" + camel_parts = [part.capitalize() for part in namespace] + return f"{prefix}{''.join(camel_parts)}SeriesAccessor" + + +def render_accessor(bq_modules: list[data_models.BQModule]) -> tuple[str, str, str]: + """ + Returns the content for core accessor, pandas accessor and BF accessor + """ + + namespaces = set() + for bq_module in bq_modules: + for i in range(len(bq_module.namespace) + 1): + namespaces.add(bq_module.namespace[:i]) + + sorted_namespaces = sorted(list(namespaces), key=lambda ns: (len(ns), ns)) + + accessors = [] + accessor_lookup_table = {} + for namespace in sorted_namespaces: + accessor = data_models.Accessor( + class_name=_create_accessor_class_name(namespace), + bigframes_class_name=_create_accessor_class_name( + namespace, prefix="Bigframes" + ), + pandas_class_name=_create_accessor_class_name(namespace, prefix="Pandas"), + is_root=len(namespace) == 0, + description=( + f"Series accessor for BigQuery {'.'.join(namespace)} functions." + if namespace + else "Series accessor for BigQuery functions." + ), + children=[], + functions=[], + ) + accessors.append(accessor) + accessor_lookup_table[namespace] = accessor + + # Establish parent-child relations + if len(namespace) > 0: + accessor.prop_name = namespace[-1] + parent_namespace = namespace[:-1] + accessor_lookup_table[parent_namespace].children.append(accessor) + + # Arrange functions by namespaces + for bq_module in bq_modules: + module_parts = bq_module.module_path.parts + for bq_func in bq_module.functions: + if bq_func.series_accessor_arg is None: + continue + bf_func = _to_bigframes_func(bq_func) + bf_func.import_module = ( + f"bigframes.operations.googlesql.{'.'.join(module_parts)}" + ) + accessor_lookup_table[bq_module.namespace].functions.append(bf_func) + + core_content = constants.TEMPLATES["core_series_accessor"].render( + script_path=constants.SCRIPT_PATH_RELATIVE, + namespaces=accessors, + ) + + pandas_content = constants.TEMPLATES["pandas_series_accessor"].render( + script_path=constants.SCRIPT_PATH_RELATIVE, namespaces=accessors + ) + + bigframes_content = constants.TEMPLATES["bigframes_series_accessor"].render( + script_path=constants.SCRIPT_PATH_RELATIVE, namespaces=accessors + ) + + return core_content, pandas_content, bigframes_content diff --git a/packages/bigframes/scripts/bigquery_generator/yaml_parser.py b/packages/bigframes/scripts/bigquery_generator/yaml_parser.py new file mode 100644 index 000000000000..277e6431ce94 --- /dev/null +++ b/packages/bigframes/scripts/bigquery_generator/yaml_parser.py @@ -0,0 +1,86 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pathlib +import re +from typing import Any + +import yaml + +from . import constants, data_models + + +def _to_snake_case(name: str) -> str: + # Replace dots with underscores + name = name.replace(".", "_") + # Handle CamelCase to snake_case + name = re.sub(r"(? data_models.BQFuncArg: + return data_models.BQFuncArg( + name=arg_data["name"], + value=arg_data["value"], + optional=arg_data["optional"], + keyword_only=arg_data["keyword_only"], + ) + + +def _parse_func_impl(impl_data: Any) -> data_models.BQFuncImpl: + return data_models.BQFuncImpl( + args=[_parse_func_arg(arg) for arg in impl_data["args"]], + return_type=impl_data["return"], + ) + + +def _parse_bq_func( + func_data: Any, module_name: str, is_global: bool +) -> data_models.BQFunc: + op_base_name = _to_snake_case(func_data["name"]) + + if not is_global and op_base_name.startswith(module_name + "_"): + op_base_name = op_base_name[len(module_name) + 1 :] + + return data_models.BQFunc( + name=func_data["name"], + op_base_name=op_base_name, + description=func_data["description"], + impls=[_parse_func_impl(impl) for impl in func_data["impls"]], + series_accessor_arg=func_data.get("series_accessor_arg", None), + ) + + +def parse_yaml(yaml_file: pathlib.Path) -> data_models.BQModule: + print(f"Parsing {yaml_file}...") + + with open(yaml_file, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + functions = [] + module_path = yaml_file.relative_to(constants.DATA_DIR).with_suffix("") + is_global = "global_namespace" in module_path.parts + if isinstance(data, dict) and "scalar_functions" in data: + functions = [ + _parse_bq_func(func_data, module_path.name, is_global) + for func_data in data["scalar_functions"] + ] + + return data_models.BQModule( + yaml_file=yaml_file, + functions=functions, + ) diff --git a/packages/bigframes/scripts/generate_bigframes_bigquery.py b/packages/bigframes/scripts/generate_bigframes_bigquery.py index 0381d168329f..c2a3c0ab813a 100755 --- a/packages/bigframes/scripts/generate_bigframes_bigquery.py +++ b/packages/bigframes/scripts/generate_bigframes_bigquery.py @@ -23,688 +23,22 @@ # limitations under the License. import pathlib -import re -import subprocess +import sys -import jinja2 -import yaml +scripts_dir = pathlib.Path(__file__).parent +if str(scripts_dir) not in sys.path: + sys.path.insert(0, str(scripts_dir)) -SCRIPTS_DIRECTORY = pathlib.Path(__file__).parent.absolute() -PACKAGE_ROOT = SCRIPTS_DIRECTORY.parent -CODE_ROOT = PACKAGE_ROOT / "bigframes" -SCRIPT_PATH_RELATIVE = pathlib.Path(__file__).relative_to(PACKAGE_ROOT) - -# Directory containing the YAML files -DATA_DIR = SCRIPTS_DIRECTORY / "data" / "sql-functions" -# Directory where the generated Python files will be placed -OUTPUT_DIR = CODE_ROOT / "operations" / "googlesql" -# Directory where the generated test files will be placed -TEST_OUTPUT_DIR = PACKAGE_ROOT / "tests" / "unit" / "bigquery" / "generated" -# Directory containing the Jinja2 templates -TEMPLATE_DIR = SCRIPTS_DIRECTORY / "templates" - - -RUFF_COMMON_ARGS = [ - "--target-version=py310", - "--line-length=88", -] -RUFF_CHECK_ARGS = [ - "ruff", - "check", - "--select", - "I,F", - "--fix", -] + RUFF_COMMON_ARGS -RUFF_FORMAT_ARGS = [ - "ruff", - "format", -] + RUFF_COMMON_ARGS - - -DTYPE_MAP = { - "binary": "dtypes.BYTES_DTYPE", - "string": "dtypes.STRING_DTYPE", - "int64": "dtypes.INT_DTYPE", - "i64": "dtypes.INT_DTYPE", - "float64": "dtypes.FLOAT_DTYPE", - "fp64": "dtypes.FLOAT_DTYPE", - "bool": "dtypes.BOOL_DTYPE", - "boolean": "dtypes.BOOL_DTYPE", - "geography": "dtypes.GEO_DTYPE", - "json": "dtypes.JSON_DTYPE", - "date": "dtypes.DATE_DTYPE", - "time": "dtypes.TIME_DTYPE", - "datetime": "dtypes.DATETIME_DTYPE", - "timestamp": "dtypes.TIMESTAMP_DTYPE", - "decimal<38,9>": "dtypes.NUMERIC_DTYPE", - "decimal<76,38>": "dtypes.BIGNUMERIC_DTYPE", -} - -PY_TYPE_MAP = { - "binary": "bytes", - "string": "str", - "int64": "int", - "i64": "int", - "float64": "float", - "fp64": "float", - "bool": "bool", - "boolean": "bool", - "geography": "Any", - "json": "Any", - "date": "datetime.date", - "time": "datetime.time", - "datetime": "datetime.datetime", - "timestamp": "datetime.datetime", - "struct": "dict", - "decimal<38,9>": "decimal.Decimal", - "decimal<76,38>": "decimal.Decimal", - "interval_day": "datetime.timedelta", -} - -YAML_TYPE_TO_COL = { - "binary": "bytes_col", - "string": "string_col", - "int64": "int64_col", - "i64": "int64_col", - "float64": "float64_col", - "fp64": "float64_col", - "bool": "bool_col", - "boolean": "bool_col", - "geography": "geography_col", - "date": "date_col", - "time": "time_col", - "datetime": "datetime_col", - "timestamp": "timestamp_col", - "decimal<38,9>": "numeric_col", - "decimal<76,38>": "bignumeric_col", -} - -_PYTHON_BUILTINS = { - "abs", - "all", - "any", - "ascii", - "bin", - "bool", - "breakpoint", - "bytearray", - "bytes", - "callable", - "chr", - "classmethod", - "compile", - "complex", - "delattr", - "dict", - "dir", - "divmod", - "enumerate", - "eval", - "exec", - "filter", - "float", - "format", - "frozenset", - "getattr", - "globals", - "hasattr", - "hash", - "help", - "hex", - "id", - "input", - "int", - "isinstance", - "issubclass", - "iter", - "len", - "list", - "locals", - "map", - "max", - "memoryview", - "min", - "next", - "object", - "oct", - "open", - "ord", - "pow", - "print", - "property", - "range", - "repr", - "reversed", - "round", - "set", - "setattr", - "slice", - "sorted", - "staticmethod", - "str", - "sum", - "super", - "tuple", - "type", - "vars", - "zip", -} - - -def to_snake_case(name): - # Replace dots with underscores - name = name.replace(".", "_") - # Handle CamelCase to snake_case - name = re.sub(r"(?"): - inner = yaml_type[5:-1] - return inner in DTYPE_MAP - return False - - -def _get_concrete_type_expr(yaml_type): - if yaml_type in DTYPE_MAP: - return DTYPE_MAP[yaml_type] - if yaml_type.startswith("list<") and yaml_type.endswith(">"): - inner = yaml_type[5:-1] - return f"dtypes.list_type({DTYPE_MAP[inner]})" - raise ValueError(f"Not a concrete type: {yaml_type}") - - -def _validate_types(impls): - for impl in impls: - for arg in impl["args"]: - val = arg["value"] - if val == "any1": - continue - if val.startswith("list<") and val.endswith(">"): - inner = val[5:-1] - if inner != "any1" and inner not in DTYPE_MAP: - raise ValueError(f"Unsupported inner type: {inner}") - continue - if val == "struct": - continue - if val not in DTYPE_MAP: - raise ValueError(f"Unsupported type: {val}") - - ret = impl["return"] - if ret == "any1": - continue - if ret.startswith("list<") and ret.endswith(">"): - inner = ret[5:-1] - if inner != "any1" and inner not in DTYPE_MAP: - raise ValueError(f"Unsupported inner type: {inner}") - continue - if ret not in DTYPE_MAP: - raise ValueError(f"Unsupported type: {ret}") - - -def _generate_signature_def(python_name, impls, sql_name, template): - for impl in impls: - uses_any1 = False - if "any1" in str(impl["return"]): - uses_any1 = True - for arg in impl["args"]: - if "any1" in str(arg["value"]): - uses_any1 = True - impl["uses_any1"] = uses_any1 - - return_types = {impl["return"] for impl in impls} - - # Optimization: if all impls return the same concrete type, - # we can skip type checking and just return that type. - if len(return_types) == 1: - ret_type = next(iter(return_types)) - if _is_concrete_type(ret_type): - sig_expr = f"lambda *args: {_get_concrete_type_expr(ret_type)}" - return sig_expr, None - - _validate_types(impls) - - func_name = f"_{python_name.upper()}_SIG" - max_args = max(len(impl["args"]) for impl in impls) - - rendered = template.render( - func_name=func_name, - max_args=max_args, - impls=impls, - sql_name=sql_name, - dtype_map=DTYPE_MAP, - ) - - return func_name, rendered - - -def _get_func_args(args_by_name, arg_order): - func_args = [] - for name in arg_order: - arg_info = args_by_name[name] - types = [PY_TYPE_MAP.get(t, "Any") for t in sorted(arg_info["types"])] + [ - "Literal[sentinels.Sentinel.ARGUMENT_DEFAULT]" - ] - type_hint = ( - "Union[" + ", ".join(sorted(set(types))) + "]" - if len(types) > 1 - else types[0] - ) - default = "sentinels.Sentinel.ARGUMENT_DEFAULT" if arg_info["optional"] else "" - func_args.append( - { - "name": name, - "type_hint": type_hint, - "default": default, - } - ) - - # Clean up default values for mandatory args - # In Python, mandatory args come first. - for arg in func_args: - if not arg.get("default"): - arg.pop("default", None) - - return func_args - - -def _get_test_args(args_by_name, arg_order): - test_args = [] - for name in arg_order: - arg_info = args_by_name[name] - some_type = sorted(arg_info["types"])[0] - col_name = YAML_TYPE_TO_COL.get(some_type, "string_col") - test_args.append({"col_name": col_name}) - return test_args - - -def parse_scalar_functions(data, module_name, signature_def_template, is_global=False): - ops_list = [] - functions_list = [] - - if "scalar_functions" not in data: - return ops_list, functions_list - - for func_data in data["scalar_functions"]: - sql_name = func_data["name"] - python_name = to_snake_case(sql_name) - if not is_global and python_name.startswith(module_name + "_"): - python_name = python_name[len(module_name) + 1 :] - - op_base_name = python_name - if python_name in _PYTHON_BUILTINS: - python_name = python_name + "_" - - internal_op_name = f"_{op_base_name.upper()}_OP" - - # Aggregate args across impls - args_by_name, arg_order = _collect_args(func_data["impls"]) - - # Build ArgSpecs - arg_specs = _build_arg_specs(args_by_name, arg_order) - arg_specs_str = ", ".join(arg_specs) - if len(arg_specs) == 1: - arg_specs_str += "," - - # Determine return dtype - sig_name, sig_def = _generate_signature_def( - op_base_name, - func_data["impls"], - sql_name, - signature_def_template, - ) - - ops_list.append( - { - "internal_name": internal_op_name, - "sql_name": sql_name.upper(), - "arg_specs": arg_specs_str, - "signature": sig_name, - "signature_definition": sig_def, - } - ) - - # Function args - func_args = _get_func_args(args_by_name, arg_order) - - # Test args - test_args = _get_test_args(args_by_name, arg_order) - - # Read series_accessor_arg - series_accessor_arg = func_data.get("series_accessor_arg") - - functions_list.append( - { - "name": python_name, - "op_name": internal_op_name, - "description": func_data["description"], - "args": func_args, - "test_args": test_args, - "series_accessor_arg": series_accessor_arg, - } - ) - - return ops_list, functions_list - - -def run_ruff(path: pathlib.Path): - import sys - - subprocess.run( - [sys.executable, "-m", "ruff"] - + RUFF_CHECK_ARGS[1:] - + [ - str(path), - ], - check=True, - ) - - subprocess.run( - [sys.executable, "-m", "ruff"] - + RUFF_FORMAT_ARGS[1:] - + [ - str(path), - ], - check=True, - ) - - -def ensure_init_py(directory: pathlib.Path, limit_dir: pathlib.Path, license_template): - """Ensures __init__.py exists in the directory and its parents up to limit_dir.""" - curr = directory - while curr != limit_dir and curr != curr.parent: - init_file = curr / "__init__.py" - if not init_file.exists(): - print(f" Creating {init_file}") - content = license_template.render() - with open(init_file, "w") as f: - f.write(content) - run_ruff(init_file) - curr = curr.parent - - -def process_yaml_file(yaml_file, templates): - print(f"Processing {yaml_file}...") - with open(yaml_file, "r") as f: - data = yaml.safe_load(f) - - rel_path = yaml_file.relative_to(DATA_DIR) - module_path = rel_path.with_suffix("") - module_name = module_path.name - output_file = OUTPUT_DIR.joinpath(module_path).with_suffix(".py") - - is_global = "global_namespace" in module_path.parts - namespace = get_namespace(yaml_file) - - if not data or not isinstance(data, dict) or "scalar_functions" not in data: - # If the file is empty or has no functions, just create the namespace. - return [{"namespace": namespace}] - - ops_list, functions_list = parse_scalar_functions( - data, - module_name, - templates["signature_def"], - is_global=is_global, - ) - - # Render and write - output_file.parent.mkdir(parents=True, exist_ok=True) - ensure_init_py(output_file.parent, OUTPUT_DIR.parent, templates["license"]) - yaml_file_relative = yaml_file.relative_to(PACKAGE_ROOT) - content = templates["operation"].render( - yaml_path=yaml_file_relative, - script_path=SCRIPT_PATH_RELATIVE, - ops=ops_list, - functions=functions_list, - ) - with open(output_file, "w") as f: - f.write(content) - - run_ruff(output_file) - print(f" Generated {output_file}") - - # Render and write test - import_path = "bigframes.operations.googlesql." + ".".join(module_path.parts) - test_output_file = TEST_OUTPUT_DIR.joinpath( - module_path.with_name(f"test_{module_path.name}") - ).with_suffix(".py") - - test_output_file.parent.mkdir(parents=True, exist_ok=True) - ensure_init_py( - test_output_file.parent, TEST_OUTPUT_DIR.parent, templates["license"] - ) - test_content = templates["test_operation"].render( - yaml_path=yaml_file_relative, - script_path=SCRIPT_PATH_RELATIVE, - import_path=import_path, - short_name=module_path.name, - is_global=is_global, - functions=functions_list, - ) - with open(test_output_file, "w") as f: - f.write(test_content) - - run_ruff(test_output_file) - print(f" Generated {test_output_file}") - - # Collect functions for Series accessor - accessor_functions = [] - for func in functions_list: - if func.get("series_accessor_arg"): - import_module = ( - f"bigframes.operations.googlesql.{'.'.join(module_path.parts)}" - ) - accessor_functions.append( - { - "name": func["name"], - "import_module": import_module, - "namespace": namespace, - "description": func["description"], - "args": func["args"], - "series_accessor_arg": func["series_accessor_arg"], - } - ) - - return accessor_functions - - -def get_namespace(yaml_file: pathlib.Path) -> tuple[str, ...] | None: - rel_path = yaml_file.relative_to(DATA_DIR) - parts = rel_path.with_suffix("").parts - if "global_namespace" in parts: - return None - return parts - - -def get_class_name(ns_tuple: tuple[str, ...], prefix: str = "") -> str: - if not ns_tuple: - return f"{prefix}BigQuerySeriesAccessor" - camel_parts = [part.capitalize() for part in ns_tuple] - return f"{prefix}{''.join(camel_parts)}SeriesAccessor" - - -def generate_series_accessors(functions: list[dict], templates: dict): - print("Generating Series accessors...") - # Find all active namespaces - active_namespaces = set() - for func in functions: - ns = func["namespace"] or () - for i in range(len(ns) + 1): - active_namespaces.add(ns[:i]) - - # Sort namespaces by depth so parents come first - sorted_namespaces = sorted(list(active_namespaces), key=len) - - # Build namespace definitions - ns_defs = [] - ns_by_tuple = {} - for ns in sorted_namespaces: - class_name = get_class_name(ns) - bf_class_name = get_class_name(ns, prefix="Bigframes") - pd_class_name = get_class_name(ns, prefix="Pandas") - - ns_def = { - "ns_tuple": ns, - "class_name": class_name, - "bigframes_class_name": bf_class_name, - "pandas_class_name": pd_class_name, - "is_root": len(ns) == 0, - "description": ( - f"Series accessor for BigQuery {'.'.join(ns)} functions." - if ns - else "Series accessor for BigQuery functions." - ), - "children": [], - "functions": [], - } - ns_defs.append(ns_def) - ns_by_tuple[ns] = ns_def - - # Populate functions - for func in functions: - if "name" in func: - ns = func["namespace"] or () - ns_by_tuple[ns]["functions"].append(func) - - # Populate children properties - for ns in sorted_namespaces: - if len(ns) > 0: - parent_ns = ns[:-1] - parent_def = ns_by_tuple[parent_ns] - child_def = ns_by_tuple[ns] - parent_def["children"].append( - { - "prop_name": ns[-1], - "class_name": child_def["class_name"], - "bigframes_class_name": child_def["bigframes_class_name"], - "pandas_class_name": child_def["pandas_class_name"], - } - ) - - # Render and write core - core_output_file = CODE_ROOT / "extensions" / "core" / "series_accessor.py" - core_output_file.parent.mkdir(parents=True, exist_ok=True) - ensure_init_py(core_output_file.parent, CODE_ROOT, templates["license"]) - core_content = templates["core_series_accessor"].render( - script_path=SCRIPT_PATH_RELATIVE, - namespaces=ns_defs, - ) - with open(core_output_file, "w") as f: - f.write(core_content) - run_ruff(core_output_file) - print(f" Generated {core_output_file}") - - # Render and write bigframes - bf_output_file = CODE_ROOT / "extensions" / "bigframes" / "series_accessor.py" - bf_output_file.parent.mkdir(parents=True, exist_ok=True) - ensure_init_py(bf_output_file.parent, CODE_ROOT, templates["license"]) - bf_content = templates["bigframes_series_accessor"].render( - script_path=SCRIPT_PATH_RELATIVE, - namespaces=ns_defs, - ) - with open(bf_output_file, "w") as f: - f.write(bf_content) - run_ruff(bf_output_file) - print(f" Generated {bf_output_file}") - - # Render and write pandas - pd_output_file = CODE_ROOT / "extensions" / "pandas" / "series_accessor.py" - pd_output_file.parent.mkdir(parents=True, exist_ok=True) - ensure_init_py(pd_output_file.parent, CODE_ROOT, templates["license"]) - pd_content = templates["pandas_series_accessor"].render( - script_path=SCRIPT_PATH_RELATIVE, - namespaces=ns_defs, - ) - with open(pd_output_file, "w") as f: - f.write(pd_content) - run_ruff(pd_output_file) - print(f" Generated {pd_output_file}") +from bigquery_generator import constants, file_generator, yaml_parser def main(): - templates = load_templates() + modules = [] - all_accessor_functions = [] - for yaml_file in sorted(DATA_DIR.glob("**/*.yaml")): - accessor_funcs = process_yaml_file(yaml_file, templates) - all_accessor_functions.extend(accessor_funcs) + for yaml_file in sorted(constants.DATA_DIR.glob("**/*.yaml")): + modules.append(yaml_parser.parse_yaml(yaml_file)) - if all_accessor_functions: - generate_series_accessors(all_accessor_functions, templates) + file_generator.generate(modules) if __name__ == "__main__":