From 0e71905c62ecf0cd1be45b5099eec47caa39fa34 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 27 Jul 2026 15:55:30 +0800 Subject: [PATCH 1/2] [FLINK-40260][python] Add Process Table Function APIs to PyFlink Add Python PTF definitions and the Java planner placeholder with explicit type inference. Expose Table API entry points for registered Python and Java PTF calls. Generated-by: Codex (GPT-5) --- flink-python/pyflink/table/__init__.py | 14 +- flink-python/pyflink/table/table.py | 90 +++++- .../pyflink/table/table_environment.py | 18 ++ .../tests/test_environment_completeness.py | 1 - .../tests/test_process_table_function_api.py | 94 ++++++ .../test_process_table_function_definition.py | 241 ++++++++++++++ .../table/tests/test_table_completeness.py | 27 +- .../test_table_environment_completeness.py | 1 - flink-python/pyflink/table/udf.py | 305 +++++++++++++++++- .../TestJavaProcessTableFunctions.java | 55 ++++ .../python/PythonProcessTableFunction.java | 216 +++++++++++++ .../PythonProcessTableFunctionTest.java | 126 ++++++++ 12 files changed, 1176 insertions(+), 12 deletions(-) create mode 100644 flink-python/pyflink/table/tests/test_process_table_function_api.py create mode 100644 flink-python/pyflink/table/tests/test_process_table_function_definition.py create mode 100644 flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/TestJavaProcessTableFunctions.java create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonProcessTableFunction.java create mode 100644 flink-table/flink-table-common/src/test/java/org/apache/flink/table/functions/python/PythonProcessTableFunctionTest.java diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py index 6fc10c9393bf7c..4ab2c057ca25f6 100644 --- a/flink-python/pyflink/table/__init__.py +++ b/flink-python/pyflink/table/__init__.py @@ -40,6 +40,8 @@ Base interface for user-defined scalar function. - :class:`TableFunction` Base interface for user-defined table function. + - :class:`ProcessTableFunction` + Base interface for user-defined process table function. - :class:`AggregateFunction` Base interface for user-defined aggregate function. - :class:`TableAggregateFunction` @@ -124,8 +126,8 @@ from pyflink.table.schema import Schema from pyflink.table.sql_dialect import SqlDialect from pyflink.table.statement_set import StatementSet -from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, Table, \ - WindowGroupedTable +from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, \ + PartitionedTable, Table, WindowGroupedTable from pyflink.table.table_config import TableConfig from pyflink.table.table_descriptor import TableDescriptor, FormatDescriptor from pyflink.table.table_environment import (TableEnvironment, StreamTableEnvironment) @@ -134,7 +136,8 @@ from pyflink.table.table_schema import TableSchema from pyflink.table.types import DataTypes, UserDefinedType, Row, RowKind from pyflink.table.udf import FunctionContext, ScalarFunction, TableFunction, AggregateFunction, \ - TableAggregateFunction, AsyncScalarFunction + TableAggregateFunction, AsyncScalarFunction, ProcessTableFunction, \ + ProcessTableFunctionArgument, ProcessTableFunctionArgumentTrait, ProcessTableFunctionState __all__ = [ 'TableEnvironment', @@ -147,8 +150,13 @@ 'GroupWindowedTable', 'OverWindowedTable', 'WindowGroupedTable', + 'PartitionedTable', 'ScalarFunction', 'TableFunction', + 'ProcessTableFunction', + 'ProcessTableFunctionArgument', + 'ProcessTableFunctionArgumentTrait', + 'ProcessTableFunctionState', 'AggregateFunction', 'TableAggregateFunction', 'AsyncScalarFunction', diff --git a/flink-python/pyflink/table/table.py b/flink-python/pyflink/table/table.py index e7898e8752d1fa..b431d162556d16 100644 --- a/flink-python/pyflink/table/table.py +++ b/flink-python/pyflink/table/table.py @@ -43,7 +43,8 @@ from pyflink.util.java_utils import to_jarray from pyflink.util.java_utils import to_j_explain_detail_arr -__all__ = ['Table', 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', 'WindowGroupedTable'] +__all__ = ['Table', 'PartitionedTable', 'GroupedTable', 'GroupWindowedTable', + 'OverWindowedTable', 'WindowGroupedTable'] @PublicEvolving() @@ -156,6 +157,46 @@ def alias(self, field: str, *fields: str) -> 'Table': extra_fields = to_jarray(gateway.jvm.String, fields) return Table(get_method(self._j_table, "as")(field, extra_fields), self._t_env) + def partition_by(self, *fields: Expression) -> 'PartitionedTable': + """ + Partitions this table for a subsequent process table function call. + + :param fields: Partitioning expressions. + :return: A partitioned table for configuring and invoking a process table function. + + .. versionadded:: 2.4.0 + """ + return PartitionedTable( + self._j_table.partitionBy(to_expression_jarray(fields)), self._t_env) + + def as_argument(self, name: str) -> Expression: + """ + Converts this table into a named table argument for a process table function call. + + :param name: Name of the table argument declared by the function. + :return: A named table argument. + + .. versionadded:: 2.4.0 + """ + return Expression(self._j_table.asArgument(name)) + + def process(self, path: str, *arguments: Expression) -> 'Table': + """ + Calls a registered process table function with this table as its first table argument. + + :param path: Path of the registered process table function. + :param arguments: Additional named arguments of the function. + :return: The result table. + + .. versionadded:: 2.4.0 + """ + if not isinstance(path, str): + raise TypeError("Process table functions must be called by a registered name.") + gateway = get_gateway() + j_arguments = to_jarray( + gateway.jvm.Object, [_get_java_expression(argument) for argument in arguments]) + return Table(self._j_table.process(path, j_arguments), self._t_env) + def filter(self, predicate: Expression[bool]) -> 'Table': """ Filters out elements that don't pass the filter predicate. Similar to a SQL WHERE @@ -1286,6 +1327,53 @@ def from_changelog(self, *arguments: Expression) -> 'Table': return Table(self._j_table.fromChangelog(to_expression_jarray(arguments)), self._t_env) +@PublicEvolving() +class PartitionedTable(object): + """ + A table argument with partitioning and optional ordering metadata for process table functions. + + .. versionadded:: 2.4.0 + """ + + def __init__(self, j_partitioned_table, t_env): + self._j_partitioned_table = j_partitioned_table + self._t_env = t_env + + def order_by(self, *fields: Expression) -> 'PartitionedTable': + """ + Defines the ordering of rows within each partition. + + :param fields: Ordering expressions. + :return: A partitioned table with ordering metadata. + """ + return PartitionedTable( + self._j_partitioned_table.orderBy(to_expression_jarray(fields)), self._t_env) + + def as_argument(self, name: str) -> Expression: + """ + Converts this partitioned table into a named table argument. + + :param name: Name of the table argument declared by the function. + :return: A named table argument. + """ + return Expression(self._j_partitioned_table.asArgument(name)) + + def process(self, path: str, *arguments: Expression) -> Table: + """ + Calls a registered process table function with this partitioned table. + + :param path: Path of the registered process table function. + :param arguments: Additional named arguments of the function. + :return: The result table. + """ + if not isinstance(path, str): + raise TypeError("Process table functions must be called by a registered name.") + gateway = get_gateway() + j_arguments = to_jarray( + gateway.jvm.Object, [_get_java_expression(argument) for argument in arguments]) + return Table(self._j_partitioned_table.process(path, j_arguments), self._t_env) + + @PublicEvolving() class GroupedTable(object): """ diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py index f0e4bedba174e8..0c66a64d684fca 100644 --- a/flink-python/pyflink/table/table_environment.py +++ b/flink-python/pyflink/table/table_environment.py @@ -36,6 +36,7 @@ from pyflink.serializers import BatchedSerializer, PickleSerializer from pyflink.table import Table, EnvironmentSettings, Expression, ExplainDetail, \ Module, ModuleEntry, Schema, ChangelogMode +from pyflink.table.expression import _get_java_expression from pyflink.table.catalog import Catalog, CatalogDescriptor from pyflink.table.model_descriptor import ModelDescriptor from pyflink.table.compiled_plan import CompiledPlan @@ -528,6 +529,23 @@ def from_path(self, path: str) -> Table: """ return Table(get_method(self._j_tenv, "from")(path), self) + def from_call(self, path: str, *arguments: Expression) -> Table: + """ + Creates a table from a registered process table function call. + + :param path: Path of the registered process table function. + :param arguments: Named arguments of the function. + :return: The result table. + + .. versionadded:: 2.4.0 + """ + if not isinstance(path, str): + raise TypeError("Process table functions must be called by a registered name.") + gateway = get_gateway() + j_arguments = to_jarray( + gateway.jvm.Object, [_get_java_expression(argument) for argument in arguments]) + return Table(self._j_tenv.fromCall(path, j_arguments), self) + def from_descriptor(self, descriptor: TableDescriptor) -> Table: """ Returns a Table backed by the given TableDescriptor. diff --git a/flink-python/pyflink/table/tests/test_environment_completeness.py b/flink-python/pyflink/table/tests/test_environment_completeness.py index ab9c69a0e0ae1f..02c6ae4d8ffe43 100644 --- a/flink-python/pyflink/table/tests/test_environment_completeness.py +++ b/flink-python/pyflink/table/tests/test_environment_completeness.py @@ -40,7 +40,6 @@ def excluded_methods(cls): return { 'getCompletionHints', 'fromValues', - 'fromCall', 'fromModel', # See FLINK-25986 'loadPlan', diff --git a/flink-python/pyflink/table/tests/test_process_table_function_api.py b/flink-python/pyflink/table/tests/test_process_table_function_api.py new file mode 100644 index 00000000000000..150c42fbe458bd --- /dev/null +++ b/flink-python/pyflink/table/tests/test_process_table_function_api.py @@ -0,0 +1,94 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import datetime + +from pyflink.common import Row +from pyflink.table import DataTypes, EnvironmentSettings, PartitionedTable, TableEnvironment +from pyflink.table.expressions import col, lit +from pyflink.testing.test_case_utils import PyFlinkTestCase + + +class ProcessTableFunctionAPITests(PyFlinkTestCase): + + @staticmethod + def _create_table_environment(): + table_env = TableEnvironment.create(EnvironmentSettings.in_streaming_mode()) + function_prefix = ( + "org.apache.flink.table.runtime.operators.python.process." + "TestJavaProcessTableFunctions$") + table_env.create_java_temporary_system_function( + "java_row_ptf", function_prefix + "RowSemanticFunction") + table_env.create_java_temporary_system_function( + "java_multi_ptf", function_prefix + "MultiInputFunction") + return table_env + + @staticmethod + def _create_input(table_env): + return table_env.from_elements( + [("Alice", 1, datetime.datetime(2026, 1, 1, 0, 0, 0))], + DataTypes.ROW([ + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("score", DataTypes.INT()), + DataTypes.FIELD("ts", DataTypes.TIMESTAMP(3)), + ])) + + def test_table_process_calls_registered_java_function(self): + table_env = self._create_table_environment() + orders = self._create_input(table_env) + + result = orders.process("java_row_ptf", lit(42).as_argument("increment")) + + self.assertEqual(["out"], result.get_schema().get_field_names()) + self.assertIn("ProcessTableFunction", result.explain()) + with result.execute().collect() as rows: + self.assertEqual([Row("Alice:42")], list(rows)) + + def test_from_call_accepts_multiple_partitioned_table_arguments(self): + table_env = self._create_table_environment() + orders = self._create_input(table_env) + profiles = self._create_input(table_env) + + partitioned_orders = orders.partition_by(col("name")) + self.assertIsInstance(partitioned_orders, PartitionedTable) + ordered_orders = partitioned_orders.order_by(col("ts")) + self.assertIsInstance(ordered_orders, PartitionedTable) + result = table_env.from_call( + "java_multi_ptf", + partitioned_orders.as_argument("in1"), + profiles.partition_by(col("name")).as_argument("in2"), + ) + + self.assertEqual(["name", "name0", "out"], result.get_schema().get_field_names()) + self.assertIn("ProcessTableFunction", result.explain()) + + def test_process_requires_registered_function_name(self): + table_env = self._create_table_environment() + orders = self._create_input(table_env) + + with self.assertRaisesRegex(TypeError, "registered name"): + orders.process(object()) + with self.assertRaisesRegex(TypeError, "registered name"): + orders.partition_by(col("name")).process(object()) + with self.assertRaisesRegex(TypeError, "registered name"): + table_env.from_call(object(), orders.as_argument("in1")) + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/flink-python/pyflink/table/tests/test_process_table_function_definition.py b/flink-python/pyflink/table/tests/test_process_table_function_definition.py new file mode 100644 index 00000000000000..28f2112272909d --- /dev/null +++ b/flink-python/pyflink/table/tests/test_process_table_function_definition.py @@ -0,0 +1,241 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +from pyflink.common import Duration, Row +from pyflink.table import DataTypes, EnvironmentSettings, TableEnvironment +from pyflink.table.udf import ( + ProcessTableFunction, + ProcessTableFunctionArgument, + ProcessTableFunctionArgumentTrait as Trait, + ProcessTableFunctionState, + udptf, +) +from pyflink.testing.test_case_utils import PyFlinkTestCase + + +class Tokenize(ProcessTableFunction): + + def eval(self, ctx, event, separator): + for token in event.text.split(separator): + yield Row(token) + + +class CountWithTimeout(ProcessTableFunction): + + def eval(self, ctx, memory, event): + memory["count"] = (memory.count or 0) + 1 + yield Row(memory.count) + + def on_timer(self, ctx, memory): + yield Row(memory.count) + + +class ProcessTableFunctionDefinitionTests(PyFlinkTestCase): + + @staticmethod + def _result_type(): + return DataTypes.ROW([DataTypes.FIELD("result", DataTypes.STRING())]) + + @staticmethod + def _memory_state(): + return ProcessTableFunctionState.value( + "memory", + DataTypes.ROW([DataTypes.FIELD("count", DataTypes.BIGINT())]), + ttl=Duration.of_days(1), + ) + + def test_creates_stateless_function_with_ordered_arguments(self): + function = udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + java_function = function._java_user_defined_function() + arguments = java_function.getTypeInference(None).getStaticArguments().get() + + self.assertEqual("PythonProcessTableFunction", java_function.getClass().getSimpleName()) + self.assertEqual(["event", "separator"], [argument.getName() for argument in arguments]) + self.assertFalse(java_function.hasOnTimer()) + + table_env = TableEnvironment.create(EnvironmentSettings.in_streaming_mode()) + table_env.create_temporary_system_function("tokenize", function) + self.assertIn("tokenize", table_env.list_user_defined_functions()) + + def test_creates_stateful_timer_function(self): + function = udptf( + CountWithTimeout(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", traits={Trait.SET_SEMANTIC_TABLE}), + ], + states=[self._memory_state()], + result_type=DataTypes.ROW([DataTypes.FIELD("count", DataTypes.BIGINT())]), + ) + + java_function = function._java_user_defined_function() + + self.assertTrue(java_function.hasOnTimer()) + self.assertEqual( + ["memory"], + list(java_function.getTypeInference(None).getStateTypeStrategies().keySet()), + ) + self.assertEqual( + 86_400_000, + java_function.getStateTimeToLive()[0].toMillis(), + ) + + def test_rejects_invalid_callback_signatures(self): + class InvalidEval(ProcessTableFunction): + def eval(self, ctx, separator, event): + yield Row(separator) + + class InvalidTimer(ProcessTableFunction): + def eval(self, ctx, memory, event): + yield Row(memory.count) + + def on_timer(self, memory, ctx): + yield Row(memory.count) + + with self.assertRaisesRegex(ValueError, r"Invalid eval\(\) signature"): + udptf( + InvalidEval(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + with self.assertRaisesRegex(ValueError, r"Invalid on_timer\(\) signature"): + udptf( + InvalidTimer(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", traits={Trait.SET_SEMANTIC_TABLE}), + ], + states=[self._memory_state()], + result_type=self._result_type(), + ) + + def test_requires_exactly_one_table_argument(self): + with self.assertRaisesRegex(ValueError, "exactly one table argument"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.scalar("event", DataTypes.STRING()), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + with self.assertRaisesRegex(ValueError, "exactly one table argument"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.table("separator"), + ], + result_type=self._result_type(), + ) + + def test_state_and_timer_require_set_semantics(self): + with self.assertRaisesRegex(ValueError, "State requires"): + udptf( + CountWithTimeout(), + arguments=[ProcessTableFunctionArgument.table("event")], + states=[self._memory_state()], + result_type=self._result_type(), + ) + + class StatelessTimer(ProcessTableFunction): + def eval(self, ctx, event): + yield Row(event) + + def on_timer(self, ctx): + yield Row("timer") + + with self.assertRaisesRegex(ValueError, "Timers require"): + udptf( + StatelessTimer(), + arguments=[ProcessTableFunctionArgument.table("event")], + result_type=self._result_type(), + ) + + def test_rejects_pass_through_with_timer(self): + with self.assertRaisesRegex(ValueError, "pass-through"): + udptf( + CountWithTimeout(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", + traits={ + Trait.SET_SEMANTIC_TABLE, + Trait.PASS_COLUMNS_THROUGH, + }, + ), + ], + states=[self._memory_state()], + result_type=self._result_type(), + ) + + def test_rejects_update_traits_in_first_version(self): + for update_trait in ( + Trait.SUPPORT_UPDATES, + Trait.REQUIRE_UPDATE_BEFORE, + Trait.REQUIRE_FULL_DELETE): + with self.subTest(update_trait=update_trait): + with self.assertRaisesRegex(ValueError, "do not support updating inputs"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", + traits={Trait.SET_SEMANTIC_TABLE, update_trait}, + ), + ProcessTableFunctionArgument.scalar( + "separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + def test_validates_argument_state_and_result_types(self): + with self.assertRaisesRegex(ValueError, "both row and set semantics"): + ProcessTableFunctionArgument.table( + "event", + traits={Trait.ROW_SEMANTIC_TABLE, Trait.SET_SEMANTIC_TABLE}, + ) + with self.assertRaisesRegex(TypeError, "state must use a ROW"): + ProcessTableFunctionState.value("memory", DataTypes.BIGINT()) + with self.assertRaisesRegex(TypeError, "result_type must be a ROW"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=DataTypes.STRING(), + ) + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/flink-python/pyflink/table/tests/test_table_completeness.py b/flink-python/pyflink/table/tests/test_table_completeness.py index feca7e63b1a90f..311c027122285e 100644 --- a/flink-python/pyflink/table/tests/test_table_completeness.py +++ b/flink-python/pyflink/table/tests/test_table_completeness.py @@ -17,7 +17,7 @@ ################################################################################ from pyflink.testing.test_case_utils import PythonAPICompletenessTestCase, PyFlinkTestCase -from pyflink.table import Table +from pyflink.table import PartitionedTable, Table class TableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): @@ -39,9 +39,6 @@ def excluded_methods(cls): return { 'createTemporalTableFunction', 'getQueryOperation', - 'asArgument', - 'process', - 'partitionBy', } @classmethod @@ -56,6 +53,28 @@ def java_method_name(cls, python_method_name): return {'alias': 'as'}.get(python_method_name, python_method_name) +class PartitionedTableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): + """ + Tests whether the Python :class:`PartitionedTable` is consistent with + Java `org.apache.flink.table.api.PartitionedTable`. + """ + + @classmethod + def python_class(cls): + return PartitionedTable + + @classmethod + def java_class(cls): + return "org.apache.flink.table.api.PartitionedTable" + + @classmethod + def excluded_methods(cls): + return { + 'fromChangelog', + 'toChangelog', + } + + if __name__ == '__main__': import unittest diff --git a/flink-python/pyflink/table/tests/test_table_environment_completeness.py b/flink-python/pyflink/table/tests/test_table_environment_completeness.py index 2e0cc5c2b390c2..98f1d47bf04a13 100644 --- a/flink-python/pyflink/table/tests/test_table_environment_completeness.py +++ b/flink-python/pyflink/table/tests/test_table_environment_completeness.py @@ -43,7 +43,6 @@ def excluded_methods(cls): "registerTable", "from", "registerFunction", - "fromCall", "fromModel", } diff --git a/flink-python/pyflink/table/udf.py b/flink-python/pyflink/table/udf.py index 92bd180d1899e1..3f06f93cb5119b 100644 --- a/flink-python/pyflink/table/udf.py +++ b/flink-python/pyflink/table/udf.py @@ -16,9 +16,21 @@ # limitations under the License. ################################################################################ import abc +import enum import functools import inspect -from typing import Union, List, Type, Callable, TypeVar, Generic, Iterable +from typing import ( + Union, + List, + Type, + Callable, + TypeVar, + Generic, + Iterable, + Optional, + Sequence, + Set, +) from pyflink.java_gateway import get_gateway from pyflink.metrics import MetricGroup @@ -28,7 +40,9 @@ from pyflink.util.api_stability_decorators import PublicEvolving, Internal __all__ = ['FunctionContext', 'AggregateFunction', 'ScalarFunction', 'TableFunction', - 'TableAggregateFunction', 'AsyncScalarFunction', 'udf', 'udtf', 'udaf', 'udtaf'] + 'TableAggregateFunction', 'AsyncScalarFunction', 'ProcessTableFunction', + 'ProcessTableFunctionArgument', 'ProcessTableFunctionState', + 'ProcessTableFunctionArgumentTrait', 'udf', 'udtf', 'udptf', 'udaf', 'udtaf'] @PublicEvolving() @@ -232,6 +246,129 @@ def eval(self, *args): pass +@PublicEvolving() +class ProcessTableFunction(UserDefinedFunction): + """ + Base interface for a Python user-defined process table function. + + A process table function consumes table and scalar arguments and can emit zero, one, or + multiple rows. Stateful functions receive their declared state entries as mutable + :class:`~pyflink.common.Row` objects before the call arguments. + + .. versionadded:: 2.4.0 + """ + + @abc.abstractmethod + def eval(self, ctx, *args): + """ + Processes an input row and yields zero, one, or multiple result rows. + """ + pass + + def on_timer(self, ctx, *states): + """ + Processes a firing event-time timer and yields zero, one, or multiple result rows. + """ + return () + + +@PublicEvolving() +class ProcessTableFunctionArgumentTrait(enum.Enum): + """ + Traits that describe the semantics of a process table function table argument. + + .. versionadded:: 2.4.0 + """ + + ROW_SEMANTIC_TABLE = 'ROW_SEMANTIC_TABLE' + SET_SEMANTIC_TABLE = 'SET_SEMANTIC_TABLE' + PASS_COLUMNS_THROUGH = 'PASS_COLUMNS_THROUGH' + SUPPORT_UPDATES = 'SUPPORT_UPDATES' + REQUIRE_ON_TIME = 'REQUIRE_ON_TIME' + OPTIONAL_PARTITION_BY = 'OPTIONAL_PARTITION_BY' + REQUIRE_UPDATE_BEFORE = 'REQUIRE_UPDATE_BEFORE' + REQUIRE_FULL_DELETE = 'REQUIRE_FULL_DELETE' + + +@PublicEvolving() +class ProcessTableFunctionArgument(object): + """ + Declaration of a scalar or table argument of a process table function. + + .. versionadded:: 2.4.0 + """ + + def __init__(self, name: str, data_type: Optional[DataType], is_table: bool, + traits: Set[ProcessTableFunctionArgumentTrait]): + if not isinstance(name, str) or not name: + raise ValueError("The argument name must be a non-empty string.") + self.name = name + self.data_type = data_type + self.is_table = is_table + self.traits = frozenset(traits) + + @staticmethod + def scalar(name: str, data_type: DataType) -> 'ProcessTableFunctionArgument': + """ + Declares a scalar argument with an explicit data type. + """ + if not isinstance(data_type, DataType): + raise TypeError("A scalar argument data type must be a DataType.") + return ProcessTableFunctionArgument(name, data_type, False, set()) + + @staticmethod + def table(name: str, + traits: Optional[Set[ProcessTableFunctionArgumentTrait]] = None, + data_type: Optional[DataType] = None) -> 'ProcessTableFunctionArgument': + """ + Declares a polymorphic or explicitly typed table argument. + """ + if data_type is not None and not isinstance(data_type, DataType): + raise TypeError("A table argument data type must be a DataType.") + normalized_traits = set( + traits or {ProcessTableFunctionArgumentTrait.ROW_SEMANTIC_TABLE}) + if not all(isinstance(t, ProcessTableFunctionArgumentTrait) for t in normalized_traits): + raise TypeError( + "Table argument traits must be ProcessTableFunctionArgumentTrait values.") + row_semantics = ProcessTableFunctionArgumentTrait.ROW_SEMANTIC_TABLE + set_semantics = ProcessTableFunctionArgumentTrait.SET_SEMANTIC_TABLE + if row_semantics in normalized_traits and set_semantics in normalized_traits: + raise ValueError("A table argument cannot have both row and set semantics.") + if row_semantics not in normalized_traits and set_semantics not in normalized_traits: + normalized_traits.add(row_semantics) + return ProcessTableFunctionArgument(name, data_type, True, normalized_traits) + + +@PublicEvolving() +class ProcessTableFunctionState(object): + """ + Declaration of a keyed value state entry of a process table function. + + .. versionadded:: 2.4.0 + """ + + def __init__(self, name: str, data_type: DataType, ttl=None): + if not isinstance(name, str) or not name: + raise ValueError("The state name must be a non-empty string.") + from pyflink.table.types import RowType + if not isinstance(data_type, RowType): + raise TypeError("Process table function state must use a ROW data type.") + if ttl is not None: + from pyflink.common import Duration + if not isinstance(ttl, Duration): + raise TypeError("State TTL must be a pyflink.common.Duration.") + self.name = name + self.data_type = data_type + self.ttl = ttl + + @staticmethod + def value(name: str, data_type: DataType, ttl=None) -> 'ProcessTableFunctionState': + """ + Declares a ROW value state entry. + """ + return ProcessTableFunctionState(name, data_type, ttl) + + T = TypeVar('T') ACC = TypeVar('ACC') @@ -676,6 +813,65 @@ def _create_delegate_function(self) -> UserDefinedFunction: return DelegationTableFunction(self._func) +class UserDefinedProcessTableFunctionWrapper(UserDefinedFunctionWrapper): + """ + Wrapper for a Python user-defined process table function. + """ + + def __init__(self, func: ProcessTableFunction, + arguments: Sequence[ProcessTableFunctionArgument], + states: Sequence[ProcessTableFunctionState], result_type: DataType, + deterministic=None, name=None): + super(UserDefinedProcessTableFunctionWrapper, self).__init__( + func, None, "general", deterministic, name) + self._arguments = tuple(arguments) + self._states = tuple(states) + self._result_type = result_type + self._has_on_timer = func.__class__.on_timer is not ProcessTableFunction.on_timer + + def _create_judf(self, serialized_func, j_input_types, j_function_kind): + gateway = get_gateway() + argument_names = java_utils.to_jarray( + gateway.jvm.String, [argument.name for argument in self._arguments]) + argument_types = java_utils.to_jarray( + gateway.jvm.DataType, + [_to_java_data_type(argument.data_type) if argument.data_type is not None else None + for argument in self._arguments]) + table_arguments = java_utils.to_jarray( + gateway.jvm.boolean, [argument.is_table for argument in self._arguments]) + argument_traits = java_utils.to_jarray( + gateway.jvm.String, + [','.join(sorted(trait.value for trait in argument.traits)) + for argument in self._arguments]) + state_names = java_utils.to_jarray( + gateway.jvm.String, [state.name for state in self._states]) + state_types = java_utils.to_jarray( + gateway.jvm.DataType, [_to_java_data_type(state.data_type) for state in self._states]) + state_ttls = java_utils.to_jarray( + gateway.jvm.java.time.Duration, + [state.ttl._j_duration if state.ttl is not None else None for state in self._states]) + + PythonProcessTableFunction = gateway.jvm \ + .org.apache.flink.table.functions.python.PythonProcessTableFunction + return PythonProcessTableFunction( + self._name, + bytearray(serialized_func), + argument_names, + argument_types, + table_arguments, + argument_traits, + state_names, + state_types, + state_ttls, + _to_java_data_type(self._result_type), + self._deterministic, + self._has_on_timer, + _get_python_env()) + + def _create_delegate_function(self) -> UserDefinedFunction: + raise TypeError("A process table function must extend ProcessTableFunction.") + + class UserDefinedAggregateFunctionWrapper(UserDefinedFunctionWrapper): """ Wrapper for Python user-defined aggregate function or user-defined table aggregate function. @@ -776,6 +972,11 @@ def _create_udtf(f, input_types, result_types, deterministic, name): return UserDefinedTableFunctionWrapper(f, input_types, result_types, deterministic, name) +def _create_udptf(f, arguments, states, result_type, deterministic, name): + return UserDefinedProcessTableFunctionWrapper( + f, arguments, states, result_type, deterministic, name) + + def _create_udaf(f, input_types, result_type, accumulator_type, func_type, deterministic, name): return UserDefinedAggregateFunctionWrapper( f, input_types, result_type, accumulator_type, func_type, deterministic, name) @@ -911,6 +1112,106 @@ def udtf(f: Union[Callable, TableFunction, Type] = None, return _create_udtf(f, input_types, result_types, deterministic, name) +def _validate_process_table_function_signature(func, method_name, expected_names): + method = getattr(func, method_name) + parameters = list(inspect.signature(method).parameters.values()) + supported_kinds = ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + invalid_kind = any(parameter.kind not in supported_kinds for parameter in parameters) + actual_names = [parameter.name for parameter in parameters] + if invalid_kind or actual_names != expected_names: + raise ValueError( + "Invalid {}() signature. Expected ({}) but found ({}).".format( + method_name, ', '.join(expected_names), ', '.join(actual_names))) + + +def udptf(f: Union[ProcessTableFunction, Type] = None, + arguments: Sequence[ProcessTableFunctionArgument] = None, + result_type: DataType = None, + states: Sequence[ProcessTableFunctionState] = None, + deterministic: bool = None, + name: str = None): + """ + Creates a Python user-defined process table function. + + The ``eval`` callback receives ``ctx``, all declared states, and all declared arguments in + exactly that order. An optional ``on_timer`` callback receives ``ctx`` and all states. + + :param f: process table function instance or class. + :param arguments: ordered scalar and table argument declarations. + :param result_type: ROW data type produced by the function. + :param states: optional ordered value state declarations. + :param deterministic: whether the function always returns the same result for the same input. + :param name: the function name. + :return: UserDefinedProcessTableFunctionWrapper or function. + + .. versionadded:: 2.4.0 + """ + if f is None: + return functools.partial( + udptf, + arguments=arguments, + result_type=result_type, + states=states, + deterministic=deterministic, + name=name) + + if inspect.isclass(f) and issubclass(f, ProcessTableFunction): + f = f() + if not isinstance(f, ProcessTableFunction): + raise TypeError("A process table function must extend ProcessTableFunction.") + if arguments is None: + raise ValueError("Process table function arguments must be declared.") + if not isinstance(arguments, (list, tuple)) or not all( + isinstance(argument, ProcessTableFunctionArgument) for argument in arguments): + raise TypeError("arguments must be an ordered list of ProcessTableFunctionArgument values.") + if states is None: + states = [] + if not isinstance(states, (list, tuple)) or not all( + isinstance(state, ProcessTableFunctionState) for state in states): + raise TypeError("states must be an ordered list of ProcessTableFunctionState values.") + + from pyflink.table.types import RowType + if not isinstance(result_type, RowType): + raise TypeError("result_type must be a ROW DataType.") + + table_arguments = [argument for argument in arguments if argument.is_table] + if len(table_arguments) != 1: + raise ValueError("A Python process table function must declare exactly one table argument.") + + names = [state.name for state in states] + [argument.name for argument in arguments] + if len(names) != len(set(names)): + raise ValueError("Process table function state and argument names must be unique.") + + table_argument = table_arguments[0] + unsupported_update_traits = { + ProcessTableFunctionArgumentTrait.SUPPORT_UPDATES, + ProcessTableFunctionArgumentTrait.REQUIRE_UPDATE_BEFORE, + ProcessTableFunctionArgumentTrait.REQUIRE_FULL_DELETE, + } + if table_argument.traits.intersection(unsupported_update_traits): + raise ValueError("Python process table functions do not support updating inputs.") + + set_semantics = ProcessTableFunctionArgumentTrait.SET_SEMANTIC_TABLE + if states and set_semantics not in table_argument.traits: + raise ValueError("State requires a table argument with set semantics.") + + state_names = [state.name for state in states] + argument_names = [argument.name for argument in arguments] + _validate_process_table_function_signature( + f, 'eval', ['ctx'] + state_names + argument_names) + if f.__class__.on_timer is not ProcessTableFunction.on_timer: + if set_semantics not in table_argument.traits: + raise ValueError("Timers require a table argument with set semantics.") + if ProcessTableFunctionArgumentTrait.PASS_COLUMNS_THROUGH in table_argument.traits: + raise ValueError("Timers do not support pass-through columns.") + _validate_process_table_function_signature(f, 'on_timer', ['ctx'] + state_names) + + return _create_udptf(f, arguments, states, result_type, deterministic, name) + + def udaf(f: Union[Callable, AggregateFunction, Type] = None, input_types: Union[List[DataType], DataType, str, List[str]] = None, result_type: Union[DataType, str] = None, accumulator_type: Union[DataType, str] = None, diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/TestJavaProcessTableFunctions.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/TestJavaProcessTableFunctions.java new file mode 100644 index 00000000000000..ef7d07324f641f --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/TestJavaProcessTableFunctions.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.ArgumentTrait; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.functions.ProcessTableFunction; +import org.apache.flink.types.Row; + +/** Java process table functions used by the PyFlink API tests. */ +public final class TestJavaProcessTableFunctions { + + private TestJavaProcessTableFunctions() {} + + /** A PTF with one row-semantic table argument. */ + @DataTypeHint("ROW<`out` STRING>") + public static class RowSemanticFunction extends ProcessTableFunction { + public void eval( + @ArgumentHint(ArgumentTrait.ROW_SEMANTIC_TABLE) Row input, Integer increment) { + collect(Row.of(String.format("%s:%s", input.getFieldAs(0), increment))); + } + } + + /** A PTF with two set-semantic table arguments. */ + @DataTypeHint("ROW<`out` STRING>") + public static class MultiInputFunction extends ProcessTableFunction { + public void eval( + Context context, + @ArgumentHint(ArgumentTrait.SET_SEMANTIC_TABLE) Row in1, + @ArgumentHint({ + ArgumentTrait.SET_SEMANTIC_TABLE, + ArgumentTrait.OPTIONAL_PARTITION_BY + }) + Row in2) { + collect(Row.of(String.format("%s:%s", in1.getFieldAs(0), in2.getFieldAs(0)))); + } + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonProcessTableFunction.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonProcessTableFunction.java new file mode 100644 index 00000000000000..402939cf7c56b4 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonProcessTableFunction.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.functions.python; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.functions.ProcessTableFunction; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.StaticArgument; +import org.apache.flink.table.types.inference.StaticArgumentTrait; +import org.apache.flink.table.types.inference.TypeInference; +import org.apache.flink.table.types.inference.TypeStrategies; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; + +/** Planner placeholder for a Python user-defined process table function. */ +@Internal +public class PythonProcessTableFunction extends ProcessTableFunction + implements PythonFunction { + + private static final long serialVersionUID = 1L; + + private final String name; + private final byte[] serializedProcessTableFunction; + private final String[] argumentNames; + private final DataType[] argumentDataTypes; + private final boolean[] tableArguments; + private final String[] argumentTraits; + private final String[] stateNames; + private final DataType[] stateDataTypes; + private final Duration[] stateTimeToLive; + private final DataType resultType; + private final boolean deterministic; + private final boolean hasOnTimer; + private final PythonEnv pythonEnv; + + public PythonProcessTableFunction( + String name, + byte[] serializedProcessTableFunction, + String[] argumentNames, + DataType[] argumentDataTypes, + boolean[] tableArguments, + String[] argumentTraits, + String[] stateNames, + DataType[] stateDataTypes, + Duration[] stateTimeToLive, + DataType resultType, + boolean deterministic, + boolean hasOnTimer, + PythonEnv pythonEnv) { + this.name = Preconditions.checkNotNull(name); + this.serializedProcessTableFunction = + Preconditions.checkNotNull(serializedProcessTableFunction); + this.argumentNames = Preconditions.checkNotNull(argumentNames); + this.argumentDataTypes = Preconditions.checkNotNull(argumentDataTypes); + this.tableArguments = Preconditions.checkNotNull(tableArguments); + this.argumentTraits = Preconditions.checkNotNull(argumentTraits); + this.stateNames = Preconditions.checkNotNull(stateNames); + this.stateDataTypes = Preconditions.checkNotNull(stateDataTypes); + this.stateTimeToLive = Preconditions.checkNotNull(stateTimeToLive); + this.resultType = Preconditions.checkNotNull(resultType); + this.deterministic = deterministic; + this.hasOnTimer = hasOnTimer; + this.pythonEnv = Preconditions.checkNotNull(pythonEnv); + validateMetadata(); + } + + public void eval(Object... args) { + throw new UnsupportedOperationException( + "This method is a placeholder and should not be called."); + } + + public void onTimer(Object... states) { + throw new UnsupportedOperationException( + "This method is a placeholder and should not be called."); + } + + @Override + public byte[] getSerializedPythonFunction() { + return serializedProcessTableFunction; + } + + @Override + public PythonEnv getPythonEnv() { + return pythonEnv; + } + + @Override + public boolean isDeterministic() { + return deterministic; + } + + public boolean hasOnTimer() { + return hasOnTimer; + } + + public String[] getArgumentNames() { + return argumentNames; + } + + public DataType[] getArgumentDataTypes() { + return argumentDataTypes; + } + + public boolean[] getTableArguments() { + return tableArguments; + } + + public String[] getArgumentTraits() { + return argumentTraits; + } + + public String[] getStateNames() { + return stateNames; + } + + public DataType[] getStateDataTypes() { + return stateDataTypes; + } + + public Duration[] getStateTimeToLive() { + return stateTimeToLive; + } + + public DataType getResultType() { + return resultType; + } + + @Override + public TypeInference getTypeInference(DataTypeFactory typeFactory) { + final List staticArguments = new ArrayList<>(); + for (int i = 0; i < argumentNames.length; i++) { + if (tableArguments[i]) { + final EnumSet traits = parseTraits(argumentTraits[i]); + final @Nullable DataType dataType = argumentDataTypes[i]; + if (dataType == null) { + staticArguments.add( + StaticArgument.table(argumentNames[i], Row.class, false, traits)); + } else { + staticArguments.add( + StaticArgument.table(argumentNames[i], dataType, false, traits)); + } + } else { + staticArguments.add( + StaticArgument.scalar(argumentNames[i], argumentDataTypes[i], false)); + } + } + + final LinkedHashMap stateStrategies = new LinkedHashMap<>(); + for (int i = 0; i < stateNames.length; i++) { + stateStrategies.put( + stateNames[i], + StateTypeStrategy.of( + TypeStrategies.explicit(stateDataTypes[i]), stateTimeToLive[i])); + } + + return TypeInference.newBuilder() + .staticArguments(staticArguments) + .stateTypeStrategies(stateStrategies) + .outputTypeStrategy(TypeStrategies.explicit(resultType)) + .build(); + } + + @Override + public String toString() { + return name; + } + + private void validateMetadata() { + Preconditions.checkArgument( + argumentNames.length == argumentDataTypes.length + && argumentNames.length == tableArguments.length + && argumentNames.length == argumentTraits.length, + "Argument metadata must have equal lengths."); + Preconditions.checkArgument( + stateNames.length == stateDataTypes.length + && stateNames.length == stateTimeToLive.length, + "State metadata must have equal lengths."); + } + + private static EnumSet parseTraits(String serializedTraits) { + final EnumSet traits = EnumSet.noneOf(StaticArgumentTrait.class); + if (serializedTraits.isEmpty()) { + return traits; + } + for (String trait : serializedTraits.split(",")) { + traits.add(StaticArgumentTrait.valueOf(trait)); + } + return traits; + } +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/functions/python/PythonProcessTableFunctionTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/functions/python/PythonProcessTableFunctionTest.java new file mode 100644 index 00000000000000..6145fa3bba47b7 --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/functions/python/PythonProcessTableFunctionTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.functions.python; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.StaticArgument; +import org.apache.flink.table.types.inference.StaticArgumentTrait; +import org.apache.flink.table.types.inference.TypeInference; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link PythonProcessTableFunction}. */ +class PythonProcessTableFunctionTest { + + private static final DataType INPUT_TYPE = + DataTypes.ROW(DataTypes.FIELD("name", DataTypes.STRING())); + private static final DataType STATE_TYPE = + DataTypes.ROW(DataTypes.FIELD("count", DataTypes.BIGINT())); + private static final DataType RESULT_TYPE = + DataTypes.ROW(DataTypes.FIELD("result", DataTypes.STRING())); + + @Test + void testCreatesOrderedTypeInference() { + final PythonProcessTableFunction function = createFunction(); + + final TypeInference typeInference = function.getTypeInference(null); + final List arguments = typeInference.getStaticArguments().orElseThrow(); + + assertThat(arguments).extracting(StaticArgument::getName).containsExactly("input", "limit"); + assertThat(arguments.get(0).is(StaticArgumentTrait.TABLE)).isTrue(); + assertThat(arguments.get(0).is(StaticArgumentTrait.SET_SEMANTIC_TABLE)).isTrue(); + assertThat(arguments.get(0).is(StaticArgumentTrait.REQUIRE_ON_TIME)).isFalse(); + assertThat(arguments.get(1).is(StaticArgumentTrait.SCALAR)).isTrue(); + assertThat(typeInference.getStateTypeStrategies()).containsOnlyKeys("memory"); + assertThat(function.getStateTimeToLive()).containsExactly(Duration.ofDays(1)); + assertThat(function.hasOnTimer()).isTrue(); + assertThat(function.getPythonEnv().getExecType()).isEqualTo(PythonEnv.ExecType.PROCESS); + } + + @Test + void testSupportsPolymorphicTableArgument() { + final PythonProcessTableFunction function = + new PythonProcessTableFunction( + "polymorphic", + new byte[] {1}, + new String[] {"input"}, + new DataType[] {null}, + new boolean[] {true}, + new String[] {"ROW_SEMANTIC_TABLE"}, + new String[0], + new DataType[0], + new Duration[0], + RESULT_TYPE, + true, + false, + new PythonEnv(PythonEnv.ExecType.PROCESS)); + + final StaticArgument argument = + function.getTypeInference(null).getStaticArguments().orElseThrow().get(0); + + assertThat(argument.getName()).isEqualTo("input"); + assertThat(argument.is(StaticArgumentTrait.ROW_SEMANTIC_TABLE)).isTrue(); + } + + @Test + void testRejectsInconsistentArgumentMetadata() { + assertThatThrownBy( + () -> + new PythonProcessTableFunction( + "invalid", + new byte[] {1}, + new String[] {"input"}, + new DataType[0], + new boolean[] {true}, + new String[] {"SET_SEMANTIC_TABLE"}, + new String[0], + new DataType[0], + new Duration[0], + RESULT_TYPE, + true, + false, + new PythonEnv(PythonEnv.ExecType.PROCESS))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Argument metadata must have equal lengths"); + } + + private static PythonProcessTableFunction createFunction() { + return new PythonProcessTableFunction( + "test_ptf", + new byte[] {1, 2, 3}, + new String[] {"input", "limit"}, + new DataType[] {INPUT_TYPE, DataTypes.INT()}, + new boolean[] {true, false}, + new String[] {"SET_SEMANTIC_TABLE", ""}, + new String[] {"memory"}, + new DataType[] {STATE_TYPE}, + new Duration[] {Duration.ofDays(1)}, + RESULT_TYPE, + true, + true, + new PythonEnv(PythonEnv.ExecType.PROCESS)); + } +} From 208155529de23532283e99dff6b5656bc9a7f353 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 28 Jul 2026 16:58:24 +0800 Subject: [PATCH 2/2] [FLINK-40260][python] Add Process Table Function runtime to PyFlink Introduce the Fn Execution protocol and Beam runner needed to execute Python Process Table Functions in the SDK harness. Implement worker-side eval and on_timer dispatch, remote value state with TTL, timer commands, and zero-to-many result framing. Add the Java runtime operator for argument projection, output correlation, watermark propagation, checkpoint-safe state commits, and event-time timer registration. Generalize Python timer registration through a shared handler and add unit coverage for worker, runner, timer, checkpoint, and restore behavior. Planner translation and end-to-end coverage will be added in follow-up changes. Generated-by: Codex (GPT-5) --- .../fn_execution/beam/beam_operations.py | 14 +- .../fn_execution/flink_fn_execution_pb2.py | 216 +++---- .../fn_execution/flink_fn_execution_pb2.pyi | 48 ++ .../table/process_table_function.py | 263 +++++++++ .../tests/test_process_table_function.py | 285 ++++++++++ .../pyflink/proto/flink-fn-execution.proto | 29 + ...e_function_definition.py => test_udptf.py} | 2 +- .../process/timer/TimerRegistration.java | 3 +- .../timer/TimerRegistrationAction.java | 4 +- .../timer/TimerRegistrationHandler.java | 28 + .../python/beam/BeamPythonFunctionRunner.java | 6 +- .../ProcessTableTimerRegistration.java | 117 ++++ .../PythonProcessTableFunctionOperator.java | 526 ++++++++++++++++++ .../beam/BeamProcessTableFunctionRunner.java | 153 +++++ .../ProcessTableTimerRegistrationTest.java | 265 +++++++++ .../BeamProcessTableFunctionRunnerTest.java | 91 +++ .../process/ReadableInternalTimeContext.java | 4 +- .../process/WritableInternalTimeContext.java | 4 +- 18 files changed, 1940 insertions(+), 118 deletions(-) create mode 100644 flink-python/pyflink/fn_execution/table/process_table_function.py create mode 100644 flink-python/pyflink/fn_execution/tests/test_process_table_function.py rename flink-python/pyflink/table/tests/{test_process_table_function_definition.py => test_udptf.py} (99%) create mode 100644 flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationHandler.java create mode 100644 flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistration.java create mode 100644 flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/PythonProcessTableFunctionOperator.java create mode 100644 flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunner.java create mode 100644 flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistrationTest.java create mode 100644 flink-python/src/test/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunnerTest.java diff --git a/flink-python/pyflink/fn_execution/beam/beam_operations.py b/flink-python/pyflink/fn_execution/beam/beam_operations.py index 184f519ec926fd..a27c413ee2f07f 100644 --- a/flink-python/pyflink/fn_execution/beam/beam_operations.py +++ b/flink-python/pyflink/fn_execution/beam/beam_operations.py @@ -37,6 +37,7 @@ from pyflink.fn_execution.datastream.process import operations import pyflink.fn_execution.table.async_function.operations as async_table_operations import pyflink.fn_execution.table.operations as table_operations +import pyflink.fn_execution.table.process_table_function as process_table_function # ----------------- UDF -------------------- @@ -136,6 +137,15 @@ def create_pandas_over_window_aggregate_function( def create_data_stream_keyed_process_function(factory, transform_id, transform_proto, parameter, consumers): urn = parameter.do_fn.urn + if urn == process_table_function.PROCESS_TABLE_FUNCTION_URN: + payload = proto_utils.parse_Bytes( + parameter.do_fn.payload, + flink_fn_execution_pb2.UserDefinedProcessTableFunction) + operation_cls = beam_operations.StatefulFunctionOperation \ + if payload.HasField('key_type') else beam_operations.StatelessFunctionOperation + return _create_user_defined_function_operation( + factory, transform_proto, consumers, payload, operation_cls, + process_table_function.ProcessTableFunctionOperation) payload = proto_utils.parse_Bytes( parameter.do_fn.payload, flink_fn_execution_pb2.UserDefinedDataStreamFunction) if urn == datastream_operations.DATA_STREAM_STATELESS_FUNCTION_URN: @@ -176,11 +186,11 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers, else: operator_state_backend = None - if hasattr(serialized_fn, "key_type"): + if hasattr(serialized_fn, "key_type") and serialized_fn.HasField("key_type"): # keyed operation, need to create the KeyedStateBackend. row_schema = serialized_fn.key_type.row_schema key_row_coder = FlattenRowCoder([from_proto(f.type) for f in row_schema.fields]) - if serialized_fn.HasField('group_window'): + if hasattr(serialized_fn, 'group_window') and serialized_fn.HasField('group_window'): if serialized_fn.group_window.is_time_window: window_coder = TimeWindowCoder() else: diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py index 96984f80d5c777..6bdc9c56045f55 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py @@ -31,7 +31,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf4\x06\n\x1fUserDefinedProcessTableFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12]\n\targuments\x18\x02 \x03(\x0b\x32J.org.apache.flink.fn_execution.v1.UserDefinedProcessTableFunction.Argument\x12W\n\x06states\x18\x03 \x03(\x0b\x32G.org.apache.flink.fn_execution.v1.UserDefinedProcessTableFunction.State\x12\x44\n\x08key_type\x18\x04 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x14\n\x0chas_on_timer\x18\x05 \x01(\x08\x12\x16\n\x0emetric_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fprofile_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x0b \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0c \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x1a|\n\x08\x41rgument\x12\x0c\n\x04name\x18\x01 \x01(\t\x12@\n\x04type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x10\n\x08is_table\x18\x03 \x01(\x08\x12\x0e\n\x06traits\x18\x04 \x03(\t\x1ak\n\x05State\x12\x0c\n\x04name\x18\x01 \x01(\t\x12@\n\x04type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x12\n\nttl_millis\x18\x03 \x01(\x03\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -49,108 +49,114 @@ _globals['_ASYNCOPTIONS']._serialized_end=559 _globals['_USERDEFINEDFUNCTIONS']._serialized_start=562 _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1013 - _globals['_OVERWINDOW']._serialized_start=1016 - _globals['_OVERWINDOW']._serialized_end=1365 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1157 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1365 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1368 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2147 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1633 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2147 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=1896 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=1980 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=1983 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2134 - _globals['_GROUPWINDOW']._serialized_start=2150 - _globals['_GROUPWINDOW']._serialized_end=2706 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2514 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2605 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2607 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2706 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2709 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3420 - _globals['_SCHEMA']._serialized_start=3423 - _globals['_SCHEMA']._serialized_end=5461 - _globals['_SCHEMA_MAPINFO']._serialized_start=3498 - _globals['_SCHEMA_MAPINFO']._serialized_end=3649 - _globals['_SCHEMA_TIMEINFO']._serialized_start=3651 - _globals['_SCHEMA_TIMEINFO']._serialized_end=3680 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3682 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3716 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3718 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3762 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3764 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3803 - _globals['_SCHEMA_DECIMALINFO']._serialized_start=3805 - _globals['_SCHEMA_DECIMALINFO']._serialized_end=3852 - _globals['_SCHEMA_BINARYINFO']._serialized_start=3854 - _globals['_SCHEMA_BINARYINFO']._serialized_end=3882 - _globals['_SCHEMA_VARBINARYINFO']._serialized_start=3884 - _globals['_SCHEMA_VARBINARYINFO']._serialized_end=3915 - _globals['_SCHEMA_CHARINFO']._serialized_start=3917 - _globals['_SCHEMA_CHARINFO']._serialized_end=3943 - _globals['_SCHEMA_VARCHARINFO']._serialized_start=3945 - _globals['_SCHEMA_VARCHARINFO']._serialized_end=3974 - _globals['_SCHEMA_FIELDTYPE']._serialized_start=3977 - _globals['_SCHEMA_FIELDTYPE']._serialized_end=5049 - _globals['_SCHEMA_FIELD']._serialized_start=5051 - _globals['_SCHEMA_FIELD']._serialized_end=5159 - _globals['_SCHEMA_TYPENAME']._serialized_start=5162 - _globals['_SCHEMA_TYPENAME']._serialized_end=5461 - _globals['_TYPEINFO']._serialized_start=5464 - _globals['_TYPEINFO']._serialized_end=6811 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=5958 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6097 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6100 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6284 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6193 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6284 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6286 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6366 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6368 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6398 - _globals['_TYPEINFO_TYPENAME']._serialized_start=6401 - _globals['_TYPEINFO_TYPENAME']._serialized_end=6798 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6814 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7791 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7309 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7615 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7618 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7791 - _globals['_STATEDESCRIPTOR']._serialized_start=7794 - _globals['_STATEDESCRIPTOR']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7926 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8397 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8575 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8663 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8665 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8740 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8743 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9351 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9353 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9451 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9453 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9497 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9565 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9567 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9641 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9643 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9686 - _globals['_CODERINFODESCRIPTOR']._serialized_start=9689 - _globals['_CODERINFODESCRIPTOR']._serialized_end=10698 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10282 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10356 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10358 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10425 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10427 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10496 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10498 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10577 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10579 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10651 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10653 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10685 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION']._serialized_start=1016 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION']._serialized_end=1900 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_ARGUMENT']._serialized_start=1667 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_ARGUMENT']._serialized_end=1791 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_STATE']._serialized_start=1793 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_STATE']._serialized_end=1900 + _globals['_OVERWINDOW']._serialized_start=1903 + _globals['_OVERWINDOW']._serialized_end=2252 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=2044 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=2252 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=2255 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=3034 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=2520 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=3034 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=2783 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=2867 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=2870 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=3021 + _globals['_GROUPWINDOW']._serialized_start=3037 + _globals['_GROUPWINDOW']._serialized_end=3593 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=3401 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=3492 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=3494 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=3593 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=3596 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=4307 + _globals['_SCHEMA']._serialized_start=4310 + _globals['_SCHEMA']._serialized_end=6348 + _globals['_SCHEMA_MAPINFO']._serialized_start=4385 + _globals['_SCHEMA_MAPINFO']._serialized_end=4536 + _globals['_SCHEMA_TIMEINFO']._serialized_start=4538 + _globals['_SCHEMA_TIMEINFO']._serialized_end=4567 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=4569 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=4603 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=4605 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=4649 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=4651 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=4690 + _globals['_SCHEMA_DECIMALINFO']._serialized_start=4692 + _globals['_SCHEMA_DECIMALINFO']._serialized_end=4739 + _globals['_SCHEMA_BINARYINFO']._serialized_start=4741 + _globals['_SCHEMA_BINARYINFO']._serialized_end=4769 + _globals['_SCHEMA_VARBINARYINFO']._serialized_start=4771 + _globals['_SCHEMA_VARBINARYINFO']._serialized_end=4802 + _globals['_SCHEMA_CHARINFO']._serialized_start=4804 + _globals['_SCHEMA_CHARINFO']._serialized_end=4830 + _globals['_SCHEMA_VARCHARINFO']._serialized_start=4832 + _globals['_SCHEMA_VARCHARINFO']._serialized_end=4861 + _globals['_SCHEMA_FIELDTYPE']._serialized_start=4864 + _globals['_SCHEMA_FIELDTYPE']._serialized_end=5936 + _globals['_SCHEMA_FIELD']._serialized_start=5938 + _globals['_SCHEMA_FIELD']._serialized_end=6046 + _globals['_SCHEMA_TYPENAME']._serialized_start=6049 + _globals['_SCHEMA_TYPENAME']._serialized_end=6348 + _globals['_TYPEINFO']._serialized_start=6351 + _globals['_TYPEINFO']._serialized_end=7698 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=6845 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6984 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6987 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=7171 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=7080 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=7171 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=7173 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=7253 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=7255 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=7285 + _globals['_TYPEINFO_TYPENAME']._serialized_start=7288 + _globals['_TYPEINFO_TYPENAME']._serialized_end=7685 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=7701 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=8678 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=8196 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=8502 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=8505 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=8678 + _globals['_STATEDESCRIPTOR']._serialized_start=8681 + _globals['_STATEDESCRIPTOR']._serialized_end=10573 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=8813 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=10573 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=9284 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=10382 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=9462 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=9550 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=9552 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=9627 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=9630 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=10238 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=10240 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=10338 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=10340 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=10382 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=10384 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=10452 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=10454 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=10528 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=10530 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=10573 + _globals['_CODERINFODESCRIPTOR']._serialized_start=10576 + _globals['_CODERINFODESCRIPTOR']._serialized_end=11585 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=11169 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=11243 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=11245 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=11312 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=11314 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=11383 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=11385 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=11464 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=11466 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=11538 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=11540 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=11572 # @@protoc_insertion_point(module_scope) diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi index b6b77c6894609e..f16d59bac21428 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi @@ -87,6 +87,54 @@ class UserDefinedFunctions(_message.Message): runtime_context: UserDefinedDataStreamFunction.RuntimeContext def __init__(self, udfs: _Optional[_Iterable[_Union[UserDefinedFunction, _Mapping]]] = ..., metric_enabled: bool = ..., windows: _Optional[_Iterable[_Union[OverWindow, _Mapping]]] = ..., profile_enabled: bool = ..., job_parameters: _Optional[_Iterable[_Union[JobParameter, _Mapping]]] = ..., async_options: _Optional[_Union[AsyncOptions, _Mapping]] = ..., runtime_context: _Optional[_Union[UserDefinedDataStreamFunction.RuntimeContext, _Mapping]] = ...) -> None: ... +class UserDefinedProcessTableFunction(_message.Message): + __slots__ = ("payload", "arguments", "states", "key_type", "has_on_timer", "metric_enabled", "profile_enabled", "state_cache_size", "map_state_read_cache_size", "map_state_write_cache_size", "job_parameters", "runtime_context") + class Argument(_message.Message): + __slots__ = ("name", "type", "is_table", "traits") + NAME_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + IS_TABLE_FIELD_NUMBER: _ClassVar[int] + TRAITS_FIELD_NUMBER: _ClassVar[int] + name: str + type: Schema.FieldType + is_table: bool + traits: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, name: _Optional[str] = ..., type: _Optional[_Union[Schema.FieldType, _Mapping]] = ..., is_table: bool = ..., traits: _Optional[_Iterable[str]] = ...) -> None: ... + class State(_message.Message): + __slots__ = ("name", "type", "ttl_millis") + NAME_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + TTL_MILLIS_FIELD_NUMBER: _ClassVar[int] + name: str + type: Schema.FieldType + ttl_millis: int + def __init__(self, name: _Optional[str] = ..., type: _Optional[_Union[Schema.FieldType, _Mapping]] = ..., ttl_millis: _Optional[int] = ...) -> None: ... + PAYLOAD_FIELD_NUMBER: _ClassVar[int] + ARGUMENTS_FIELD_NUMBER: _ClassVar[int] + STATES_FIELD_NUMBER: _ClassVar[int] + KEY_TYPE_FIELD_NUMBER: _ClassVar[int] + HAS_ON_TIMER_FIELD_NUMBER: _ClassVar[int] + METRIC_ENABLED_FIELD_NUMBER: _ClassVar[int] + PROFILE_ENABLED_FIELD_NUMBER: _ClassVar[int] + STATE_CACHE_SIZE_FIELD_NUMBER: _ClassVar[int] + MAP_STATE_READ_CACHE_SIZE_FIELD_NUMBER: _ClassVar[int] + MAP_STATE_WRITE_CACHE_SIZE_FIELD_NUMBER: _ClassVar[int] + JOB_PARAMETERS_FIELD_NUMBER: _ClassVar[int] + RUNTIME_CONTEXT_FIELD_NUMBER: _ClassVar[int] + payload: bytes + arguments: _containers.RepeatedCompositeFieldContainer[UserDefinedProcessTableFunction.Argument] + states: _containers.RepeatedCompositeFieldContainer[UserDefinedProcessTableFunction.State] + key_type: Schema.FieldType + has_on_timer: bool + metric_enabled: bool + profile_enabled: bool + state_cache_size: int + map_state_read_cache_size: int + map_state_write_cache_size: int + job_parameters: _containers.RepeatedCompositeFieldContainer[JobParameter] + runtime_context: UserDefinedDataStreamFunction.RuntimeContext + def __init__(self, payload: _Optional[bytes] = ..., arguments: _Optional[_Iterable[_Union[UserDefinedProcessTableFunction.Argument, _Mapping]]] = ..., states: _Optional[_Iterable[_Union[UserDefinedProcessTableFunction.State, _Mapping]]] = ..., key_type: _Optional[_Union[Schema.FieldType, _Mapping]] = ..., has_on_timer: bool = ..., metric_enabled: bool = ..., profile_enabled: bool = ..., state_cache_size: _Optional[int] = ..., map_state_read_cache_size: _Optional[int] = ..., map_state_write_cache_size: _Optional[int] = ..., job_parameters: _Optional[_Iterable[_Union[JobParameter, _Mapping]]] = ..., runtime_context: _Optional[_Union[UserDefinedDataStreamFunction.RuntimeContext, _Mapping]] = ...) -> None: ... + class OverWindow(_message.Message): __slots__ = ("window_type", "lower_boundary", "upper_boundary") class WindowType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): diff --git a/flink-python/pyflink/fn_execution/table/process_table_function.py b/flink-python/pyflink/fn_execution/table/process_table_function.py new file mode 100644 index 00000000000000..2f5e14071ee762 --- /dev/null +++ b/flink-python/pyflink/fn_execution/table/process_table_function.py @@ -0,0 +1,263 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import datetime + +from pyflink.common import Instant, Row, Time +from pyflink.datastream.state import StateTtlConfig +from pyflink.fn_execution import pickle +from pyflink.fn_execution.coders import from_proto +from pyflink.fn_execution.table.operations import BaseOperation +from pyflink.fn_execution.utils.operation_utils import normalize_table_function_result + + +PROCESS_TABLE_FUNCTION_URN = "flink:transform:process_table_function:v1" + +REGISTER_ANONYMOUS = 0 +REGISTER_NAMED = 1 +DELETE_ANONYMOUS = 2 +DELETE_NAMED = 3 +CLEAR_ALL = 4 +TRIGGER = 5 + + +def _state_key(key): + return list(key) if isinstance(key, Row) else key + + +def _to_millis(value): + if isinstance(value, bool): + raise TypeError("A timer timestamp must not be a bool.") + if isinstance(value, int): + return value + if isinstance(value, Instant): + return value.to_epoch_milli() + if isinstance(value, datetime.datetime): + if value.tzinfo is not None: + raise ValueError("Timer datetime values must be UTC-naive.") + epoch = datetime.datetime(1970, 1, 1) + return int((value - epoch).total_seconds() * 1000) + raise TypeError("Time values must be int, Instant, or UTC-naive datetime.datetime.") + + +def _from_millis(value, conversion_type): + if value is None: + return None + if conversion_type is int: + return value + if conversion_type is Instant: + return Instant.of_epoch_milli(value) + if conversion_type is datetime.datetime: + return datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=value) + raise TypeError("Time context type must be int, Instant, or datetime.datetime.") + + +class _TimerCommandEmitter(object): + + def __init__(self, keyed_state_backend): + self._keyed_state_backend = keyed_state_backend + self._timer_coder_impl = None + self._output_stream = None + + def add_timer_info(self, timer_info): + self._timer_coder_impl = timer_info.timer_coder_impl + self._output_stream = timer_info.output_stream + + def emit(self, operation, timestamp=None, name=None): + if self._timer_coder_impl is None: + raise RuntimeError("Timer services are not available for this PTF call.") + from apache_beam.transforms import userstate + from apache_beam.transforms.window import GlobalWindow + + key = self._keyed_state_backend.get_current_key() + timer_key = Row(*key) if isinstance(key, list) else key + timer_data = Row(operation, timestamp, name, timer_key, None, None) + timer = userstate.Timer( + user_key=timer_data, + dynamic_timer_tag='', + windows=(GlobalWindow(),), + clear_bit=True, + fire_timestamp=None, + hold_timestamp=None, + paneinfo=None) + self._timer_coder_impl.encode_to_stream(timer, self._output_stream, True) + self._timer_coder_impl._key_coder_impl._value_coder._output_stream.maybe_flush() + + +class _ProcessTableTimeContext(object): + + def __init__(self, context, conversion_type): + if conversion_type not in (int, Instant, datetime.datetime): + raise TypeError("Time context type must be int, Instant, or datetime.datetime.") + self._context = context + self._conversion_type = conversion_type + + def time(self): + return _from_millis(self._context._time, self._conversion_type) + + def table_watermark(self): + return _from_millis(self._context._table_watermark, self._conversion_type) + + def current_watermark(self): + return _from_millis(self._context._current_watermark, self._conversion_type) + + def register_on_time(self, *args): + if len(args) == 1: + self._context._timer_emitter.emit(REGISTER_ANONYMOUS, _to_millis(args[0])) + elif len(args) == 2 and isinstance(args[0], str): + self._context._timer_emitter.emit( + REGISTER_NAMED, _to_millis(args[1]), args[0]) + else: + raise TypeError("register_on_time expects time or name, time.") + + def clear_timer(self, time_or_name): + if isinstance(time_or_name, str): + self._context._timer_emitter.emit(DELETE_NAMED, name=time_or_name) + else: + self._context._timer_emitter.emit( + DELETE_ANONYMOUS, timestamp=_to_millis(time_or_name)) + + def clear_all_timers(self): + self._context.clear_all_timers() + + +class _ProcessTableFunctionContext(object): + + def __init__(self, timer_emitter, state_names=None): + self._timer_emitter = timer_emitter + self._state_names = None if state_names is None else frozenset(state_names) + self._cleared_states = set() + self._time = None + self._table_watermark = None + self._current_watermark = None + self._current_timer = None + + def set_event(self, time, table_watermark, current_watermark, current_timer=None): + self._cleared_states.clear() + self._time = time + self._table_watermark = table_watermark + self._current_watermark = current_watermark + self._current_timer = current_timer + + def time_context(self, conversion_type): + return _ProcessTableTimeContext(self, conversion_type) + + def clear_state(self, name): + if self._state_names is not None and name not in self._state_names: + raise ValueError("Unknown state entry: %s" % name) + self._cleared_states.add(name) + + def clear_all_state(self): + self._cleared_states.add(None) + + def clear_all_timers(self): + self._timer_emitter.emit(CLEAR_ALL) + + def clear_all(self): + self.clear_all_state() + self.clear_all_timers() + + def current_timer(self): + return self._current_timer + + +class ProcessTableFunctionOperation(BaseOperation): + + def __init__(self, serialized_fn, keyed_state_backend=None): + self.keyed_state_backend = keyed_state_backend + self._function = pickle.loads(serialized_fn.payload) + self._state_specs = list(serialized_fn.states) + self._state_handles = [] + self._timer_emitter = _TimerCommandEmitter(keyed_state_backend) + self._context = _ProcessTableFunctionContext( + self._timer_emitter, [state.name for state in self._state_specs]) + super(ProcessTableFunctionOperation, self).__init__(serialized_fn) + + def generate_func(self, serialized_fn): + return lambda value: (), [self._function] + + def open(self): + super(ProcessTableFunctionOperation, self).open() + if self.keyed_state_backend is None: + return + self._open_state_handles() + + def _open_state_handles(self): + for state in self._state_specs: + ttl_config = None + if state.ttl_millis > 0: + ttl_config = StateTtlConfig.new_builder( + Time.milliseconds(state.ttl_millis)).build() + self._state_handles.append(self.keyed_state_backend.get_value_state( + state.name, from_proto(state.type), ttl_config)) + + def finish(self): + super(ProcessTableFunctionOperation, self).finish() + if self.keyed_state_backend is not None: + self.keyed_state_backend.commit() + + def add_timer_info(self, timer_info): + self._timer_emitter.add_timer_info(timer_info) + + def process_element(self, value): + key, arguments, time, table_watermark, current_watermark = value + if self.keyed_state_backend is not None: + self.keyed_state_backend.set_current_key(_state_key(key)) + self._context.set_event(time, table_watermark, current_watermark) + return self._invoke(self._function.eval, list(arguments)) + + def process_timer(self, timer_data): + operation, timestamp, name, key, table_watermark, current_watermark = timer_data + if operation != TRIGGER: + raise ValueError("Unexpected PTF timer input operation: %s" % operation) + self.keyed_state_backend.set_current_key(_state_key(key)) + self._context.set_event(timestamp, table_watermark, current_watermark, name) + return self._invoke(self._function.on_timer, []) + + def _invoke(self, callback, arguments): + states = self._read_states() + + def invoke(): + completed = False + try: + results = callback(self._context, *states, *arguments) + for result in normalize_table_function_result(results): + yield result + completed = True + finally: + if completed: + self._write_states(states) + + return invoke() + + def _read_states(self): + states = [] + for spec, handle in zip(self._state_specs, self._state_handles): + value = handle.value() + if value is None: + value = Row(**{field.name: None for field in spec.type.row_schema.fields}) + states.append(value) + return states + + def _write_states(self, states): + clear_all = None in self._context._cleared_states + for spec, handle, value in zip(self._state_specs, self._state_handles, states): + if clear_all or spec.name in self._context._cleared_states or all( + field is None for field in value._values): + handle.clear() + else: + handle.update(value) diff --git a/flink-python/pyflink/fn_execution/tests/test_process_table_function.py b/flink-python/pyflink/fn_execution/tests/test_process_table_function.py new file mode 100644 index 00000000000000..c86657d4239d3a --- /dev/null +++ b/flink-python/pyflink/fn_execution/tests/test_process_table_function.py @@ -0,0 +1,285 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import datetime +import unittest +from types import SimpleNamespace +from unittest import mock + +from pyflink.common import Instant, Row +from pyflink.fn_execution.table.process_table_function import ( + CLEAR_ALL, + DELETE_ANONYMOUS, + DELETE_NAMED, + REGISTER_ANONYMOUS, + REGISTER_NAMED, + TRIGGER, + ProcessTableFunctionOperation, + _ProcessTableFunctionContext, +) + + +class _StateHandle(object): + + def __init__(self, value=None): + self._value = value + self.updated = [] + self.clear_count = 0 + + def value(self): + return self._value + + def update(self, value): + self.updated.append(value) + self._value = value + + def clear(self): + self.clear_count += 1 + self._value = None + + +class _TimerEmitter(object): + + def __init__(self): + self.commands = [] + + def emit(self, operation, timestamp=None, name=None): + self.commands.append((operation, timestamp, name)) + + +class _KeyedBackend(object): + + def __init__(self): + self.current_key = None + + def set_current_key(self, key): + self.current_key = key + + +class _StateBackend(_KeyedBackend): + + def __init__(self): + super(_StateBackend, self).__init__() + self.requests = [] + + def get_value_state(self, name, coder, ttl_config): + handle = _StateHandle() + self.requests.append((name, coder, ttl_config, handle)) + return handle + + +def _state_spec(name, *field_names): + fields = [SimpleNamespace(name=field_name) for field_name in field_names] + return SimpleNamespace( + name=name, + type=SimpleNamespace(row_schema=SimpleNamespace(fields=fields))) + + +def _state_spec_with_ttl(name, ttl_millis, *field_names): + spec = _state_spec(name, *field_names) + spec.ttl_millis = ttl_millis + return spec + + +def _operation(handle): + operation = object.__new__(ProcessTableFunctionOperation) + operation._state_specs = [_state_spec("memory", "count")] + operation._state_handles = [handle] + operation._context = _ProcessTableFunctionContext(_TimerEmitter()) + operation._context.set_event(1000, 900, 800) + return operation + + +class ProcessTableFunctionOperationTests(unittest.TestCase): + + def test_empty_state_is_injected_and_written_after_generator_completion(self): + handle = _StateHandle() + operation = _operation(handle) + + def callback(ctx, memory): + self.assertIsNone(memory.count) + memory["count"] = 1 + yield Row(1) + + self.assertEqual([[1]], list(operation._invoke(callback, []))) + self.assertEqual(1, handle.updated[0].count) + self.assertEqual(0, handle.clear_count) + + def test_explicit_clear_wins_over_state_mutation(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def callback(ctx, memory): + memory["count"] = 2 + ctx.clear_state("memory") + return None + + self.assertEqual([], list(operation._invoke(callback, []))) + self.assertEqual([], handle.updated) + self.assertEqual(1, handle.clear_count) + + def test_unknown_state_clear_is_rejected(self): + context = _ProcessTableFunctionContext(_TimerEmitter(), ["memory"]) + + with self.assertRaisesRegex(ValueError, "Unknown state entry: missing"): + context.clear_state("missing") + + def test_all_null_state_is_cleared(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def callback(ctx, memory): + memory["count"] = None + return None + + self.assertEqual([], list(operation._invoke(callback, []))) + self.assertEqual(1, handle.clear_count) + + def test_state_is_not_written_when_generator_raises(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def callback(ctx, memory): + memory["count"] = 2 + yield Row(2) + raise RuntimeError("boom") + + with self.assertRaisesRegex(RuntimeError, "boom"): + list(operation._invoke(callback, [])) + self.assertEqual([], handle.updated) + self.assertEqual(0, handle.clear_count) + + def test_multiple_states_follow_declaration_order(self): + first = _StateHandle(Row(count=1)) + second = _StateHandle() + operation = _operation(first) + operation._state_specs = [ + _state_spec("first", "count"), + _state_spec("second", "count"), + ] + operation._state_handles = [first, second] + + def callback(ctx, first_state, second_state): + first_state["count"] += 1 + second_state["count"] = 10 + return None + + self.assertEqual([], list(operation._invoke(callback, []))) + self.assertEqual(2, first.updated[0].count) + self.assertEqual(10, second.updated[0].count) + + def test_state_ttl_is_forwarded_to_remote_backend(self): + backend = _StateBackend() + operation = object.__new__(ProcessTableFunctionOperation) + operation.keyed_state_backend = backend + operation._state_specs = [ + _state_spec_with_ttl("expiring", 1234, "count"), + _state_spec_with_ttl("persistent", 0, "count"), + ] + operation._state_handles = [] + + with mock.patch( + 'pyflink.fn_execution.table.process_table_function.from_proto', + return_value='state-coder'): + operation._open_state_handles() + + self.assertEqual(["expiring", "persistent"], + [request[0] for request in backend.requests]) + self.assertEqual(["state-coder", "state-coder"], + [request[1] for request in backend.requests]) + self.assertEqual(1234, backend.requests[0][2].get_ttl().to_milliseconds()) + self.assertIsNone(backend.requests[1][2]) + self.assertEqual(2, len(operation._state_handles)) + + def test_zero_to_many_results(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def no_results(ctx, memory): + return None + + def many_results(ctx, memory): + yield Row(1) + yield Row(2) + + self.assertEqual([], list(operation._invoke(no_results, []))) + self.assertEqual([[1], [2]], list(operation._invoke(many_results, []))) + + def test_process_timer_sets_key_and_current_timer(self): + handle = _StateHandle(Row(count=3)) + backend = _KeyedBackend() + operation = _operation(handle) + operation.keyed_state_backend = backend + + class Function(object): + def on_timer(self, ctx, memory): + yield Row(memory.count, ctx.current_timer()) + + operation._function = Function() + key = Row("user-1") + results = list(operation.process_timer( + (TRIGGER, 1200, "timeout", key, 1100, 1150))) + + self.assertEqual(["user-1"], backend.current_key) + self.assertEqual([[3, "timeout"]], results) + self.assertEqual(1200, operation._context.time_context(int).time()) + + +class ProcessTableFunctionTimeContextTests(unittest.TestCase): + + def setUp(self): + self.emitter = _TimerEmitter() + self.context = _ProcessTableFunctionContext(self.emitter) + self.context.set_event(1000, 900, 800, "timeout") + + def test_time_context_conversions(self): + self.assertEqual(1000, self.context.time_context(int).time()) + self.assertEqual( + 1000, self.context.time_context(Instant).time().to_epoch_milli()) + self.assertEqual( + datetime.datetime(1970, 1, 1, 0, 0, 1), + self.context.time_context(datetime.datetime).time()) + self.assertEqual("timeout", self.context.current_timer()) + with self.assertRaises(TypeError): + self.context.time_context(str) + + def test_register_and_clear_timers(self): + time_context = self.context.time_context(int) + time_context.register_on_time(1100) + time_context.register_on_time("timeout", Instant.of_epoch_milli(1200)) + time_context.clear_timer(1100) + time_context.clear_timer("timeout") + time_context.clear_all_timers() + + self.assertEqual( + [ + (REGISTER_ANONYMOUS, 1100, None), + (REGISTER_NAMED, 1200, "timeout"), + (DELETE_ANONYMOUS, 1100, None), + (DELETE_NAMED, None, "timeout"), + (CLEAR_ALL, None, None), + ], + self.emitter.commands) + + def test_rejects_timezone_aware_datetime(self): + with self.assertRaisesRegex(ValueError, "UTC-naive"): + self.context.time_context(datetime.datetime).register_on_time( + datetime.datetime.now(datetime.timezone.utc)) + + +if __name__ == '__main__': + unittest.main() diff --git a/flink-python/pyflink/proto/flink-fn-execution.proto b/flink-python/pyflink/proto/flink-fn-execution.proto index 4ab5616b011b88..9d58707a61c87c 100644 --- a/flink-python/pyflink/proto/flink-fn-execution.proto +++ b/flink-python/pyflink/proto/flink-fn-execution.proto @@ -87,6 +87,35 @@ message UserDefinedFunctions { UserDefinedDataStreamFunction.RuntimeContext runtime_context = 7; } +// A Python user-defined process table function and its explicit runtime contract. +message UserDefinedProcessTableFunction { + message Argument { + string name = 1; + Schema.FieldType type = 2; + bool is_table = 3; + repeated string traits = 4; + } + + message State { + string name = 1; + Schema.FieldType type = 2; + int64 ttl_millis = 3; + } + + bytes payload = 1; + repeated Argument arguments = 2; + repeated State states = 3; + Schema.FieldType key_type = 4; + bool has_on_timer = 5; + bool metric_enabled = 6; + bool profile_enabled = 7; + int32 state_cache_size = 8; + int32 map_state_read_cache_size = 9; + int32 map_state_write_cache_size = 10; + repeated JobParameter job_parameters = 11; + UserDefinedDataStreamFunction.RuntimeContext runtime_context = 12; +} + // Used to describe the info of over window in pandas batch over window aggregation message OverWindow { enum WindowType { diff --git a/flink-python/pyflink/table/tests/test_process_table_function_definition.py b/flink-python/pyflink/table/tests/test_udptf.py similarity index 99% rename from flink-python/pyflink/table/tests/test_process_table_function_definition.py rename to flink-python/pyflink/table/tests/test_udptf.py index 28f2112272909d..9d0526fd3d5b73 100644 --- a/flink-python/pyflink/table/tests/test_process_table_function_definition.py +++ b/flink-python/pyflink/table/tests/test_udptf.py @@ -45,7 +45,7 @@ def on_timer(self, ctx, memory): yield Row(memory.count) -class ProcessTableFunctionDefinitionTests(PyFlinkTestCase): +class UdptfTests(PyFlinkTestCase): @staticmethod def _result_type(): diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java index dde1558b362bef..04634eb2c005a9 100644 --- a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java @@ -35,7 +35,7 @@ /** Handles the interaction with the Python worker for registering and deleting timers. */ @Internal -public final class TimerRegistration { +public final class TimerRegistration implements TimerRegistrationHandler { private final KeyedStateBackend keyedStateBackend; private final InternalTimerService internalTimerService; @@ -61,6 +61,7 @@ public TimerRegistration( this.baisWrapper = new DataInputViewStreamWrapper(bais); } + @Override public void setTimer(byte[] serializedTimerData) { try { bais.setBuffer(serializedTimerData, 0, serializedTimerData.length); diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java index 609f550067166e..3bb5335a2b5c3c 100644 --- a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java @@ -23,7 +23,7 @@ /** {@link TimerRegistrationAction} used to register Timer. */ public class TimerRegistrationAction { - private final TimerRegistration timerRegistration; + private final TimerRegistrationHandler timerRegistration; private final byte[] serializedTimerData; @@ -32,7 +32,7 @@ public class TimerRegistrationAction { private final List containingList; public TimerRegistrationAction( - TimerRegistration timerRegistration, + TimerRegistrationHandler timerRegistration, byte[] serializedTimerData, List containingList) { this.timerRegistration = timerRegistration; diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationHandler.java b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationHandler.java new file mode 100644 index 00000000000000..ce2721480445fd --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationHandler.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.operators.python.process.timer; + +import org.apache.flink.annotation.Internal; + +/** Handles timer registration commands emitted by a Python worker. */ +@Internal +public interface TimerRegistrationHandler { + + void setTimer(byte[] serializedTimerData); +} diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java index 8e5e5fdd586829..dabd3dc951b5c1 100644 --- a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java @@ -36,8 +36,8 @@ import org.apache.flink.runtime.memory.OpaqueMemoryResource; import org.apache.flink.runtime.state.KeyedStateBackend; import org.apache.flink.runtime.state.OperatorStateBackend; -import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistration; import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationAction; +import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationHandler; import org.apache.flink.streaming.api.runners.python.beam.state.BeamStateRequestHandler; import org.apache.flink.util.Preconditions; import org.apache.flink.util.ShutdownHookUtil; @@ -136,7 +136,7 @@ public abstract class BeamPythonFunctionRunner implements PythonFunctionRunner { @Nullable private final TypeSerializer namespaceSerializer; - @Nullable private final TimerRegistration timerRegistration; + @Nullable private final TimerRegistrationHandler timerRegistration; private final MemoryManager memoryManager; @@ -209,7 +209,7 @@ public BeamPythonFunctionRunner( @Nullable OperatorStateBackend operatorStateBackend, @Nullable TypeSerializer keySerializer, @Nullable TypeSerializer namespaceSerializer, - @Nullable TimerRegistration timerRegistration, + @Nullable TimerRegistrationHandler timerRegistration, MemoryManager memoryManager, double managedMemoryFraction, FlinkFnApi.CoderInfoDescriptor inputCoderDescriptor, diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistration.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistration.java new file mode 100644 index 00000000000000..c8948ac73c7748 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistration.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.runtime.state.KeyedStateBackend; +import org.apache.flink.streaming.api.operators.KeyContext; +import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationHandler; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.runtime.operators.process.WritableInternalTimeContext; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.util.FlinkRuntimeException; + +import static org.apache.flink.streaming.api.utils.PythonOperatorUtils.setCurrentKeyForStreaming; + +/** Applies process table function timer commands in the operator mailbox thread. */ +@Internal +final class ProcessTableTimerRegistration implements TimerRegistrationHandler { + + static final byte REGISTER_ANONYMOUS = 0; + static final byte REGISTER_NAMED = 1; + static final byte DELETE_ANONYMOUS = 2; + static final byte DELETE_NAMED = 3; + static final byte CLEAR_ALL = 4; + static final byte TRIGGER = 5; + + private final KeyContext keyContext; + private final KeyedStateBackend keyedStateBackend; + private final WritableInternalTimeContext timeContext; + private final TypeSerializer timerDataSerializer; + private final int keyArity; + private final ByteArrayInputStreamWithPos inputStream; + private final DataInputViewStreamWrapper inputView; + + ProcessTableTimerRegistration( + KeyContext keyContext, + KeyedStateBackend keyedStateBackend, + WritableInternalTimeContext timeContext, + TypeSerializer timerDataSerializer, + int keyArity) { + this.keyContext = keyContext; + this.keyedStateBackend = keyedStateBackend; + this.timeContext = timeContext; + this.timerDataSerializer = timerDataSerializer; + this.keyArity = keyArity; + this.inputStream = new ByteArrayInputStreamWithPos(); + this.inputView = new DataInputViewStreamWrapper(inputStream); + } + + @Override + public void setTimer(byte[] serializedTimerData) { + try { + inputStream.setBuffer(serializedTimerData, 0, serializedTimerData.length); + final RowData command = timerDataSerializer.deserialize(inputView); + final byte operation = command.getByte(0); + final RowData key = toBackendKey(command.getRow(3, keyArity)); + synchronized (keyedStateBackend) { + setBackendCurrentKey(key); + keyContext.setCurrentKey(key); + switch (operation) { + case REGISTER_ANONYMOUS: + timeContext.registerOnTime(command.getLong(1)); + break; + case REGISTER_NAMED: + timeContext.registerOnTime( + command.getString(2).toString(), command.getLong(1)); + break; + case DELETE_ANONYMOUS: + timeContext.clearTimer(command.getLong(1)); + break; + case DELETE_NAMED: + timeContext.clearTimer(command.getString(2).toString()); + break; + case CLEAR_ALL: + timeContext.clearAllTimers(); + break; + default: + throw new IllegalArgumentException( + "Unknown PTF timer operation: " + operation); + } + } + } catch (Exception e) { + throw new FlinkRuntimeException("Failed to apply a Python PTF timer command.", e); + } + } + + private RowData toBackendKey(RowData key) { + final TypeSerializer serializer = keyedStateBackend.getKeySerializer(); + if (serializer instanceof RowDataSerializer) { + return ((RowDataSerializer) serializer).toBinaryRow(key).copy(); + } + return key; + } + + private void setBackendCurrentKey(RowData key) { + setCurrentKeyForStreaming((KeyedStateBackend) keyedStateBackend, key); + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/PythonProcessTableFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/PythonProcessTableFunctionOperator.java new file mode 100644 index 00000000000000..f5d600a87ed5b5 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/PythonProcessTableFunctionOperator.java @@ -0,0 +1,526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.state.MapState; +import org.apache.flink.api.common.state.MapStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; +import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.core.memory.ManagedMemoryUseCase; +import org.apache.flink.fnexecution.v1.FlinkFnApi; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.runtime.state.KeyedStateBackend; +import org.apache.flink.runtime.state.VoidNamespace; +import org.apache.flink.runtime.state.VoidNamespaceSerializer; +import org.apache.flink.streaming.api.operators.InternalTimer; +import org.apache.flink.streaming.api.operators.InternalTimerService; +import org.apache.flink.streaming.api.operators.Triggerable; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.data.utils.JoinedRowData; +import org.apache.flink.table.functions.ProcessTableFunction; +import org.apache.flink.table.functions.python.PythonEnv; +import org.apache.flink.table.functions.python.PythonProcessTableFunction; +import org.apache.flink.table.runtime.generated.GeneratedProjection; +import org.apache.flink.table.runtime.generated.Projection; +import org.apache.flink.table.runtime.operators.process.RuntimeStateInfo; +import org.apache.flink.table.runtime.operators.process.RuntimeTableSemantics; +import org.apache.flink.table.runtime.operators.process.WritableInternalTimeContext; +import org.apache.flink.table.runtime.operators.python.AbstractOneInputPythonFunctionOperator; +import org.apache.flink.table.runtime.operators.python.utils.StreamRecordRowDataWrappingCollector; +import org.apache.flink.table.runtime.runners.python.beam.BeamProcessTableFunctionRunner; +import org.apache.flink.table.runtime.typeutils.PythonTypeUtils; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.runtime.typeutils.StringDataSerializer; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TinyIntType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.List; +import java.util.Queue; +import java.util.stream.Collectors; + +import static org.apache.flink.python.PythonOptions.MAP_STATE_READ_CACHE_SIZE; +import static org.apache.flink.python.PythonOptions.MAP_STATE_WRITE_CACHE_SIZE; +import static org.apache.flink.python.PythonOptions.PYTHON_METRIC_ENABLED; +import static org.apache.flink.python.PythonOptions.PYTHON_PROFILE_ENABLED; +import static org.apache.flink.python.PythonOptions.STATE_CACHE_SIZE; +import static org.apache.flink.python.util.ProtoUtils.createFlattenRowTypeCoderInfoDescriptorProto; +import static org.apache.flink.python.util.ProtoUtils.createRowTypeCoderInfoDescriptorProto; +import static org.apache.flink.streaming.api.utils.PythonOperatorUtils.setCurrentKeyForStreaming; + +/** Runtime operator for a single-input Python {@link ProcessTableFunction}. */ +@Internal +public final class PythonProcessTableFunctionOperator + extends AbstractOneInputPythonFunctionOperator + implements Triggerable { + + private static final long serialVersionUID = 1L; + + private static final String NAMED_TIMER_STATE_NAME = "__flink_internal_python_ptf_named_timers"; + private static final String NAMED_TIMER_SERVICE_NAME = + "__flink_internal_python_ptf_named_timer_service"; + private static final String ANONYMOUS_TIMER_SERVICE_NAME = + "__flink_internal_python_ptf_anonymous_timer_service"; + + private final PythonProcessTableFunction function; + private final RuntimeTableSemantics tableSemantics; + private final List stateInfos; + private final RowType inputType; + private final RowType argumentType; + private final RowType resultType; + private final RowType keyType; + private final GeneratedProjection argumentGeneratedProjection; + + private transient Projection argumentProjection; + private transient TypeSerializer runnerInputSerializer; + private transient TypeSerializer resultSerializer; + private transient TypeSerializer timerDataSerializer; + private transient ByteArrayInputStreamWithPos bais; + private transient DataInputViewStreamWrapper baisWrapper; + private transient ByteArrayOutputStreamWithPos baos; + private transient DataOutputViewStreamWrapper baosWrapper; + private transient RowDataSerializer inputSerializer; + private transient RowDataSerializer keySerializer; + private transient TypeSerializer stateKeySerializer; + private transient StreamRecordRowDataWrappingCollector outputCollector; + private transient Queue invocations; + private transient Invocation currentInvocation; + private transient JoinedRowData withResult; + private transient JoinedRowData withRowtime; + private transient long inputWatermark; + private transient Object keyForTimerService; + + private transient @Nullable MapState namedTimers; + private transient @Nullable InternalTimerService namedTimerService; + private transient @Nullable InternalTimerService anonymousTimerService; + private transient @Nullable WritableInternalTimeContext timeContext; + private transient @Nullable ProcessTableTimerRegistration timerRegistration; + private transient RowType runnerInputType; + private transient RowType timerDataType; + + public PythonProcessTableFunctionOperator( + Configuration config, + PythonProcessTableFunction function, + RuntimeTableSemantics tableSemantics, + List stateInfos, + RowType inputType, + RowType argumentType, + RowType resultType, + RowType keyType, + GeneratedProjection argumentGeneratedProjection) { + super(config); + this.function = function; + this.tableSemantics = tableSemantics; + this.stateInfos = stateInfos; + this.inputType = inputType; + this.argumentType = argumentType; + this.resultType = resultType; + this.keyType = keyType; + this.argumentGeneratedProjection = argumentGeneratedProjection; + } + + @Override + public void open() throws Exception { + bais = new ByteArrayInputStreamWithPos(); + baisWrapper = new DataInputViewStreamWrapper(bais); + baos = new ByteArrayOutputStreamWithPos(); + baosWrapper = new DataOutputViewStreamWrapper(baos); + runnerInputType = createRunnerInputType(); + timerDataType = createTimerDataType(); + runnerInputSerializer = PythonTypeUtils.toInternalSerializer(runnerInputType); + resultSerializer = PythonTypeUtils.toInternalSerializer(resultType); + timerDataSerializer = PythonTypeUtils.toInternalSerializer(timerDataType); + inputSerializer = new RowDataSerializer(inputType); + keySerializer = + tableSemantics.hasSetSemantics() + ? (RowDataSerializer) + (TypeSerializer) getKeyedStateBackend().getKeySerializer() + : new RowDataSerializer(keyType); + stateKeySerializer = PythonTypeUtils.toInternalSerializer(keyType); + argumentProjection = + argumentGeneratedProjection.newInstance( + Thread.currentThread().getContextClassLoader()); + outputCollector = new StreamRecordRowDataWrappingCollector(output); + invocations = new ArrayDeque<>(); + withResult = new JoinedRowData(); + withRowtime = new JoinedRowData(); + inputWatermark = Long.MIN_VALUE; + setTimerServices(); + super.open(); + } + + @Override + public void processElement(StreamRecord element) throws Exception { + final RowData input = element.getValue(); + final RowData key = + tableSemantics.hasSetSemantics() ? (RowData) getCurrentKey() : GenericRowData.of(); + final RowData prefix = createPrefix(input, key); + final Long time = extractTime(input); + invocations.add(new Invocation(prefix, time)); + + final GenericRowData runnerInput = new GenericRowData(5); + runnerInput.setField(0, key); + runnerInput.setField(1, argumentProjection.apply(input)); + runnerInput.setField(2, time); + runnerInput.setField(3, nullableWatermark(inputWatermark)); + runnerInput.setField(4, nullableWatermark(currentWatermark())); + runnerInputSerializer.serialize(runnerInput, baosWrapper); + pythonFunctionRunner.process(baos.toByteArray()); + baos.reset(); + elementCount++; + checkInvokeFinishBundleByCount(); + emitResults(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + inputWatermark = mark.getTimestamp(); + super.processWatermark(mark); + } + + @Override + public void prepareSnapshotPreBarrier(long checkpointId) throws Exception { + super.prepareSnapshotPreBarrier(checkpointId); + drainUnregisteredTimers(); + } + + @Override + public void endInput() throws Exception { + super.endInput(); + drainUnregisteredTimers(); + } + + @Override + public void onEventTime(InternalTimer timer) throws Exception { + setBackendCurrentKey(timer.getKey()); + final boolean named = timer.getNamespace() != VoidNamespace.INSTANCE; + if (named) { + namedTimers.remove((StringData) timer.getNamespace()); + } + final RowData key = timer.getKey(); + invocations.add(new Invocation(keySerializer.copy(key), timer.getTimestamp())); + + final GenericRowData timerData = new GenericRowData(6); + timerData.setField(0, ProcessTableTimerRegistration.TRIGGER); + timerData.setField(1, timer.getTimestamp()); + timerData.setField(2, named ? timer.getNamespace() : null); + timerData.setField(3, key); + timerData.setField(4, null); + timerData.setField(5, nullableWatermark(currentWatermark())); + timerDataSerializer.serialize(timerData, baosWrapper); + pythonFunctionRunner.processTimer(baos.toByteArray()); + baos.reset(); + elementCount++; + checkInvokeFinishBundleByCount(); + emitResults(); + } + + @Override + public void onProcessingTime(InternalTimer timer) {} + + @Override + public PythonFunctionRunner createPythonFunctionRunner() throws Exception { + final boolean stateful = tableSemantics.hasSetSemantics(); + return new BeamProcessTableFunctionRunner( + getContainingTask().getEnvironment(), + getRuntimeContext().getTaskInfo().getTaskName(), + createPythonEnvironmentManager(), + createFunctionProto(), + getFlinkMetricContainer(), + stateful ? getKeyedStateBackend() : null, + stateful ? stateKeySerializer : null, + timerRegistration, + getContainingTask().getEnvironment().getMemoryManager(), + getOperatorConfig() + .getManagedMemoryFractionOperatorUseCaseOfSlot( + ManagedMemoryUseCase.PYTHON, + getContainingTask().getJobConfiguration(), + getContainingTask() + .getEnvironment() + .getTaskManagerInfo() + .getConfiguration(), + getContainingTask() + .getEnvironment() + .getUserCodeClassLoader() + .asClassLoader()), + createFlattenRowTypeCoderInfoDescriptorProto( + runnerInputType, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, true), + createFlattenRowTypeCoderInfoDescriptorProto( + resultType, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, true), + timerRegistration == null + ? null + : createRowTypeCoderInfoDescriptorProto( + timerDataType, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false)); + } + + @Override + public PythonEnv getPythonEnv() { + return function.getPythonEnv(); + } + + @Override + public void emitResult(Tuple3 resultTuple) throws Exception { + if (currentInvocation == null) { + currentInvocation = invocations.remove(); + } + while (resultTuple != null) { + if (isFinishResult(resultTuple)) { + currentInvocation = null; + return; + } + bais.setBuffer(resultTuple.f1, 0, resultTuple.f2); + final RowData result = resultSerializer.deserialize(baisWrapper); + if (result.getRowKind() != RowKind.INSERT) { + throw new TableRuntimeException( + "Python process table functions support append-only output."); + } + emitResultRow(currentInvocation, result); + resultTuple = pythonFunctionRunner.pollResult(); + } + } + + @Override + public void setCurrentKey(Object key) { + keyForTimerService = key; + } + + @Override + public Object getCurrentKey() { + return keyForTimerService; + } + + private void emitResultRow(Invocation invocation, RowData result) { + withResult.replace(invocation.prefix, result).setRowKind(RowKind.INSERT); + if (shouldEmitRowtime()) { + final GenericRowData rowtime = + GenericRowData.of( + invocation.time == null + ? null + : TimestampData.fromEpochMillis(invocation.time)); + withRowtime.replace(withResult, rowtime).setRowKind(RowKind.INSERT); + outputCollector.collect(withRowtime); + } else { + outputCollector.collect(withResult); + } + } + + private RowData createPrefix(RowData input, RowData key) { + if (tableSemantics.passColumnsThrough()) { + return inputSerializer.copy(input); + } + if (tableSemantics.hasSetSemantics()) { + return keySerializer.copy(key); + } + return GenericRowData.of(); + } + + private @Nullable Long extractTime(RowData input) { + final int timeColumn = tableSemantics.timeColumn(); + if (timeColumn < 0 || input.isNullAt(timeColumn)) { + return null; + } + final LogicalType timeType = inputType.getTypeAt(timeColumn); + final int precision; + if (timeType instanceof LocalZonedTimestampType) { + precision = ((LocalZonedTimestampType) timeType).getPrecision(); + } else { + precision = + ((org.apache.flink.table.types.logical.TimestampType) timeType).getPrecision(); + } + return input.getTimestamp(timeColumn, precision).getMillisecond(); + } + + private long currentWatermark() { + if (anonymousTimerService != null) { + return anonymousTimerService.currentWatermark(); + } + return inputWatermark; + } + + private static @Nullable Long nullableWatermark(long watermark) { + return watermark == Long.MIN_VALUE ? null : watermark; + } + + private boolean shouldEmitRowtime() { + return tableSemantics.timeColumn() >= 0; + } + + private static boolean isFinishResult(Tuple3 resultTuple) { + return resultTuple.f2 == 1 && resultTuple.f1[0] == 0x00; + } + + private void setBackendCurrentKey(RowData key) { + setCurrentKeyForStreaming( + (KeyedStateBackend) (KeyedStateBackend) getKeyedStateBackend(), key); + } + + private void setTimerServices() throws Exception { + if (!tableSemantics.hasSetSemantics() || !function.hasOnTimer()) { + return; + } + final MapStateDescriptor descriptor = + new MapStateDescriptor<>( + NAMED_TIMER_STATE_NAME, + StringDataSerializer.INSTANCE, + LongSerializer.INSTANCE); + namedTimers = getKeyedStateStore().getMapState(descriptor); + namedTimerService = + getInternalTimerService( + NAMED_TIMER_SERVICE_NAME, + StringDataSerializer.INSTANCE, + (Triggerable) this); + anonymousTimerService = + getInternalTimerService( + ANONYMOUS_TIMER_SERVICE_NAME, + VoidNamespaceSerializer.INSTANCE, + (Triggerable) this); + timeContext = + new WritableInternalTimeContext( + namedTimers, namedTimerService, anonymousTimerService); + timerRegistration = + new ProcessTableTimerRegistration( + this, + getKeyedStateBackend(), + timeContext, + timerDataSerializer, + keyType.getFieldCount()); + } + + private RowType createRunnerInputType() { + return RowType.of( + new LogicalType[] { + keyType, + argumentType, + new BigIntType(true), + new BigIntType(true), + new BigIntType(true) + }, + new String[] {"key", "arguments", "time", "table_watermark", "current_watermark"}); + } + + private RowType createTimerDataType() { + return RowType.of( + new LogicalType[] { + new TinyIntType(false), + new BigIntType(true), + new VarCharType(true, VarCharType.MAX_LENGTH), + keyType, + new BigIntType(true), + new BigIntType(true) + }, + new String[] { + "operation", "timestamp", "name", "key", "table_watermark", "current_watermark" + }); + } + + private FlinkFnApi.UserDefinedProcessTableFunction createFunctionProto() { + final FlinkFnApi.UserDefinedProcessTableFunction.Builder builder = + FlinkFnApi.UserDefinedProcessTableFunction.newBuilder() + .setPayload( + com.google.protobuf.ByteString.copyFrom( + function.getSerializedPythonFunction())) + .setHasOnTimer(function.hasOnTimer()) + .setMetricEnabled(config.get(PYTHON_METRIC_ENABLED)) + .setProfileEnabled(config.get(PYTHON_PROFILE_ENABLED)) + .setStateCacheSize(config.get(STATE_CACHE_SIZE)) + .setMapStateReadCacheSize(config.get(MAP_STATE_READ_CACHE_SIZE)) + .setMapStateWriteCacheSize(config.get(MAP_STATE_WRITE_CACHE_SIZE)); + if (tableSemantics.hasSetSemantics()) { + builder.setKeyType(PythonTypeUtils.toProtoType(keyType)); + } + final String[] names = function.getArgumentNames(); + final boolean[] tables = function.getTableArguments(); + final String[] traits = function.getArgumentTraits(); + for (int i = 0; i < names.length; i++) { + final FlinkFnApi.UserDefinedProcessTableFunction.Argument.Builder argument = + FlinkFnApi.UserDefinedProcessTableFunction.Argument.newBuilder() + .setName(names[i]) + .setType(PythonTypeUtils.toProtoType(argumentType.getTypeAt(i))) + .setIsTable(tables[i]); + if (!traits[i].isEmpty()) { + argument.addAllTraits(Arrays.asList(traits[i].split(","))); + } + builder.addArguments(argument); + } + for (RuntimeStateInfo stateInfo : stateInfos) { + builder.addStates( + FlinkFnApi.UserDefinedProcessTableFunction.State.newBuilder() + .setName(stateInfo.getStateName()) + .setType( + PythonTypeUtils.toProtoType( + stateInfo.getDataType().getLogicalType())) + .setTtlMillis(stateInfo.getTimeToLive())); + } + builder.addAllJobParameters( + getRuntimeContext().getGlobalJobParameters().entrySet().stream() + .map( + entry -> + FlinkFnApi.JobParameter.newBuilder() + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build()) + .collect(Collectors.toList())); + builder.setRuntimeContext( + FlinkFnApi.UserDefinedDataStreamFunction.RuntimeContext.newBuilder() + .setTaskName(getRuntimeContext().getTaskInfo().getTaskName()) + .setTaskNameWithSubtasks( + getRuntimeContext().getTaskInfo().getTaskNameWithSubtasks()) + .setNumberOfParallelSubtasks( + getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks()) + .setMaxNumberOfParallelSubtasks( + getRuntimeContext().getTaskInfo().getMaxNumberOfParallelSubtasks()) + .setIndexOfThisSubtask( + getRuntimeContext().getTaskInfo().getIndexOfThisSubtask()) + .setAttemptNumber(getRuntimeContext().getTaskInfo().getAttemptNumber()) + .build()); + return builder.build(); + } + + private static final class Invocation { + private final RowData prefix; + private final @Nullable Long time; + + private Invocation(RowData prefix, @Nullable Long time) { + this.prefix = prefix; + this.time = time; + } + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunner.java new file mode 100644 index 00000000000000..d1e9eb4f15c791 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunner.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.runners.python.beam; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.fnexecution.v1.FlinkFnApi; +import org.apache.flink.python.env.process.ProcessPythonEnvironmentManager; +import org.apache.flink.python.metric.process.FlinkMetricContainer; +import org.apache.flink.python.util.ProtoUtils; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.memory.MemoryManager; +import org.apache.flink.runtime.state.KeyedStateBackend; +import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationHandler; +import org.apache.flink.streaming.api.runners.python.beam.BeamPythonFunctionRunner; + +import com.google.protobuf.GeneratedMessage; +import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.apache.beam.runners.core.construction.BeamUrns; +import org.apache.beam.runners.core.construction.graph.TimerReference; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.apache.flink.python.Constants.INPUT_COLLECTION_ID; +import static org.apache.flink.python.Constants.MAIN_INPUT_NAME; +import static org.apache.flink.python.Constants.MAIN_OUTPUT_NAME; +import static org.apache.flink.python.Constants.OUTPUT_COLLECTION_ID; +import static org.apache.flink.python.Constants.TIMER_ID; +import static org.apache.flink.python.Constants.TRANSFORM_ID; +import static org.apache.flink.python.Constants.WRAPPER_TIMER_CODER_ID; + +/** Beam runner adapter for Python process table functions. */ +@Internal +public final class BeamProcessTableFunctionRunner extends BeamPythonFunctionRunner { + + public static final String PROCESS_TABLE_FUNCTION_URN = + "flink:transform:process_table_function:v1"; + + private final GeneratedMessage functionProto; + private final @Nullable FlinkFnApi.CoderInfoDescriptor timerCoderDescriptor; + + public BeamProcessTableFunctionRunner( + Environment environment, + String taskName, + ProcessPythonEnvironmentManager environmentManager, + GeneratedMessage functionProto, + @Nullable FlinkMetricContainer flinkMetricContainer, + @Nullable KeyedStateBackend keyedStateBackend, + @Nullable TypeSerializer keySerializer, + @Nullable TimerRegistrationHandler timerRegistration, + MemoryManager memoryManager, + double managedMemoryFraction, + FlinkFnApi.CoderInfoDescriptor inputCoderDescriptor, + FlinkFnApi.CoderInfoDescriptor outputCoderDescriptor, + @Nullable FlinkFnApi.CoderInfoDescriptor timerCoderDescriptor) { + super( + environment, + taskName, + environmentManager, + flinkMetricContainer, + keyedStateBackend, + null, + keySerializer, + null, + timerRegistration, + memoryManager, + managedMemoryFraction, + inputCoderDescriptor, + outputCoderDescriptor, + Collections.emptyMap()); + this.functionProto = functionProto; + this.timerCoderDescriptor = timerCoderDescriptor; + } + + @Override + protected void buildTransforms(RunnerApi.Components.Builder componentsBuilder) { + final RunnerApi.ParDoPayload.Builder payload = + RunnerApi.ParDoPayload.newBuilder() + .setDoFn( + RunnerApi.FunctionSpec.newBuilder() + .setUrn(PROCESS_TABLE_FUNCTION_URN) + .setPayload( + org.apache.beam.vendor.grpc.v1p60p1.com.google + .protobuf.ByteString.copyFrom( + functionProto.toByteArray())) + .build()); + if (timerCoderDescriptor != null) { + payload.putTimerFamilySpecs( + TIMER_ID, + RunnerApi.TimerFamilySpec.newBuilder() + .setTimeDomain(RunnerApi.TimeDomain.Enum.EVENT_TIME) + .setTimerFamilyCoderId(WRAPPER_TIMER_CODER_ID) + .build()); + } + + componentsBuilder.putTransforms( + TRANSFORM_ID, + RunnerApi.PTransform.newBuilder() + .setUniqueName(TRANSFORM_ID) + .setSpec( + RunnerApi.FunctionSpec.newBuilder() + .setUrn( + BeamUrns.getUrn( + RunnerApi.StandardPTransforms.Primitives + .PAR_DO)) + .setPayload(payload.build().toByteString()) + .build()) + .putInputs(MAIN_INPUT_NAME, INPUT_COLLECTION_ID) + .putOutputs(MAIN_OUTPUT_NAME, OUTPUT_COLLECTION_ID) + .build()); + } + + @Override + protected List getTimers(RunnerApi.Components components) { + if (timerCoderDescriptor == null) { + return Collections.emptyList(); + } + final RunnerApi.ExecutableStagePayload.TimerId timerId = + RunnerApi.ExecutableStagePayload.TimerId.newBuilder() + .setTransformId(TRANSFORM_ID) + .setLocalName(TIMER_ID) + .build(); + return Collections.singletonList(TimerReference.fromTimerId(timerId, components)); + } + + @Override + protected Optional getOptionalTimerCoderProto() { + if (timerCoderDescriptor == null) { + return Optional.empty(); + } + return Optional.of(ProtoUtils.createCoderProto(timerCoderDescriptor)); + } +} diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistrationTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistrationTest.java new file mode 100644 index 00000000000000..cb12ca53cfe84b --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistrationTest.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.api.common.state.MapState; +import org.apache.flink.api.common.state.MapStateDescriptor; +import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.runtime.state.VoidNamespace; +import org.apache.flink.runtime.state.VoidNamespaceSerializer; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.InternalTimer; +import org.apache.flink.streaming.api.operators.InternalTimerService; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.Triggerable; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.operators.process.WritableInternalTimeContext; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.runtime.typeutils.StringDataSerializer; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TinyIntType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.flink.util.FlinkRuntimeException; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.flink.streaming.api.utils.PythonOperatorUtils.setCurrentKeyForStreaming; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link ProcessTableTimerRegistration}. */ +class ProcessTableTimerRegistrationTest { + + private static final RowType KEY_TYPE = + RowType.of(new LogicalType[] {new VarCharType()}, new String[] {"key"}); + private static final RowType TIMER_DATA_TYPE = + RowType.of( + new LogicalType[] { + new TinyIntType(false), + new BigIntType(true), + new VarCharType(true, VarCharType.MAX_LENGTH), + KEY_TYPE, + new BigIntType(true), + new BigIntType(true) + }, + new String[] { + "operation", + "timestamp", + "name", + "key", + "table_watermark", + "current_watermark" + }); + + @Test + void testNamedAnonymousDeleteAndClearOperations() throws Exception { + final TestTimerOperator operator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(operator, null)) { + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 10L, "replace", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 20L, "replace", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 15L, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.DELETE_ANONYMOUS, 15L, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 18L, null, "A"); + + harness.processWatermark(new Watermark(19L)); + assertThat(operator.firedTimers).containsExactly("A::18"); + + harness.processWatermark(new Watermark(20L)); + assertThat(operator.firedTimers).containsExactly("A::18", "A:replace:20"); + } + } + + @Test + void testDeleteAndClearAllAreScopedToCurrentKey() throws Exception { + final TestTimerOperator operator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(operator, null)) { + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 10L, "delete", "A"); + operator.setTimer(ProcessTableTimerRegistration.DELETE_NAMED, null, "delete", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 11L, "clear", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 12L, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.CLEAR_ALL, null, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 13L, null, "A"); + + harness.processWatermark(new Watermark(13L)); + + assertThat(operator.firedTimers).containsExactly("A::13"); + } + } + + @Test + void testTimersAreRestoredFromCheckpoint() throws Exception { + final OperatorSubtaskState snapshot; + final TestTimerOperator firstOperator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(firstOperator, null)) { + firstOperator.setTimer( + ProcessTableTimerRegistration.REGISTER_NAMED, 50L, "restored", "A"); + firstOperator.setTimer( + ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 51L, null, "A"); + snapshot = harness.snapshot(1L, 1L); + } + + final TestTimerOperator restoredOperator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(restoredOperator, snapshot)) { + harness.processWatermark(new Watermark(51L)); + + assertThat(restoredOperator.firedTimers) + .containsExactlyInAnyOrder("A:restored:50", "A::51"); + } + } + + @Test + void testRejectsUnknownTimerOperation() throws Exception { + final TestTimerOperator operator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness ignored = + createHarness(operator, null)) { + assertThatThrownBy(() -> operator.setTimer((byte) 99, null, null, "A")) + .isInstanceOf(FlinkRuntimeException.class) + .hasMessageContaining("Failed to apply a Python PTF timer command") + .hasRootCauseMessage("Unknown PTF timer operation: 99"); + } + } + + private static KeyedOneInputStreamOperatorTestHarness createHarness( + TestTimerOperator operator, OperatorSubtaskState initialState) throws Exception { + final KeySelector keySelector = + value -> GenericRowData.of(value.getString(0)); + final KeyedOneInputStreamOperatorTestHarness harness = + new KeyedOneInputStreamOperatorTestHarness<>( + operator, keySelector, InternalTypeInfo.of(KEY_TYPE), 1, 1, 0); + harness.setup(new RowDataSerializer(KEY_TYPE)); + if (initialState != null) { + harness.initializeState(initialState); + } + harness.open(); + return harness; + } + + private static final class TestTimerOperator extends AbstractStreamOperator + implements OneInputStreamOperator, Triggerable { + + private static final long serialVersionUID = 1L; + + private final List firedTimers = new ArrayList<>(); + private transient MapState namedTimers; + private transient ProcessTableTimerRegistration registration; + private transient RowDataSerializer timerDataSerializer; + private transient Object keyForTimerService; + + @Override + public void open() throws Exception { + super.open(); + timerDataSerializer = new RowDataSerializer(TIMER_DATA_TYPE); + namedTimers = + getKeyedStateStore() + .getMapState( + new MapStateDescriptor<>( + "test-named-timers", + StringDataSerializer.INSTANCE, + LongSerializer.INSTANCE)); + final InternalTimerService namedTimerService = + getInternalTimerService( + "test-named-timer-service", + StringDataSerializer.INSTANCE, + (Triggerable) this); + final InternalTimerService anonymousTimerService = + getInternalTimerService( + "test-anonymous-timer-service", + VoidNamespaceSerializer.INSTANCE, + (Triggerable) this); + registration = + new ProcessTableTimerRegistration( + this, + getKeyedStateBackend(), + new WritableInternalTimeContext( + namedTimers, namedTimerService, anonymousTimerService), + timerDataSerializer, + 1); + } + + @Override + public void processElement(StreamRecord element) {} + + @Override + public void setCurrentKey(Object key) { + keyForTimerService = key; + } + + @Override + public Object getCurrentKey() { + return keyForTimerService; + } + + @Override + public void onEventTime(InternalTimer timer) throws Exception { + setCurrentKey(timer.getKey()); + setCurrentKeyForStreaming( + (org.apache.flink.runtime.state.KeyedStateBackend) + (org.apache.flink.runtime.state.KeyedStateBackend) + getKeyedStateBackend(), + timer.getKey()); + final String timerName; + if (timer.getNamespace() == VoidNamespace.INSTANCE) { + timerName = ""; + } else { + final StringData name = (StringData) timer.getNamespace(); + namedTimers.remove(name); + timerName = name.toString(); + } + firedTimers.add( + timer.getKey().getString(0) + ":" + timerName + ":" + timer.getTimestamp()); + } + + @Override + public void onProcessingTime(InternalTimer timer) {} + + private void setTimer(byte operation, Long timestamp, String name, String key) + throws Exception { + final GenericRowData command = new GenericRowData(6); + command.setField(0, operation); + command.setField(1, timestamp); + command.setField(2, name == null ? null : StringData.fromString(name)); + command.setField(3, GenericRowData.of(StringData.fromString(key))); + command.setField(4, null); + command.setField(5, null); + + final ByteArrayOutputStreamWithPos output = new ByteArrayOutputStreamWithPos(); + timerDataSerializer.serialize(command, new DataOutputViewStreamWrapper(output)); + registration.setTimer(output.toByteArray()); + } + } +} diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunnerTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunnerTest.java new file mode 100644 index 00000000000000..2cb5ae2ac25cb7 --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunnerTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.runners.python.beam; + +import org.apache.flink.fnexecution.v1.FlinkFnApi; +import org.apache.flink.python.Constants; +import org.apache.flink.table.runtime.utils.PythonTestUtils; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.RowType; + +import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.junit.jupiter.api.Test; + +import static org.apache.flink.python.util.ProtoUtils.createFlattenRowTypeCoderInfoDescriptorProto; +import static org.apache.flink.python.util.ProtoUtils.createRowTypeCoderInfoDescriptorProto; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link BeamProcessTableFunctionRunner}. */ +class BeamProcessTableFunctionRunnerTest { + + private static final RowType DATA_TYPE = RowType.of(new BigIntType()); + + @Test + void testBuildsProcessTableFunctionTransformWithoutTimers() throws Exception { + final BeamProcessTableFunctionRunner runner = createRunner(false); + final RunnerApi.ParDoPayload payload = buildPayload(runner); + + assertThat(payload.getDoFn().getUrn()) + .isEqualTo(BeamProcessTableFunctionRunner.PROCESS_TABLE_FUNCTION_URN); + assertThat(payload.getTimerFamilySpecsMap()).isEmpty(); + assertThat(runner.getOptionalTimerCoderProto()).isEmpty(); + } + + @Test + void testDeclaresEventTimeTimerFamily() throws Exception { + final BeamProcessTableFunctionRunner runner = createRunner(true); + final RunnerApi.ParDoPayload payload = buildPayload(runner); + + assertThat(payload.getTimerFamilySpecsMap()).containsOnlyKeys(Constants.TIMER_ID); + assertThat(payload.getTimerFamilySpecsOrThrow(Constants.TIMER_ID).getTimeDomain()) + .isEqualTo(RunnerApi.TimeDomain.Enum.EVENT_TIME); + assertThat(runner.getOptionalTimerCoderProto()).isPresent(); + } + + private static RunnerApi.ParDoPayload buildPayload(BeamProcessTableFunctionRunner runner) + throws Exception { + final RunnerApi.Components.Builder components = RunnerApi.Components.newBuilder(); + runner.buildTransforms(components); + return RunnerApi.ParDoPayload.parseFrom( + components.getTransformsOrThrow(Constants.TRANSFORM_ID).getSpec().getPayload()); + } + + private static BeamProcessTableFunctionRunner createRunner(boolean withTimers) { + final FlinkFnApi.CoderInfoDescriptor dataCoder = + createFlattenRowTypeCoderInfoDescriptorProto( + DATA_TYPE, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, true); + return new BeamProcessTableFunctionRunner( + null, + "test-task", + PythonTestUtils.createTestProcessEnvironmentManager(), + FlinkFnApi.UserDefinedProcessTableFunction.getDefaultInstance(), + null, + null, + null, + null, + null, + 0.0, + dataCoder, + dataCoder, + withTimers + ? createRowTypeCoderInfoDescriptorProto( + DATA_TYPE, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false) + : null); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java index cbdadac3eb30ad..bf455bc827f51e 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java @@ -32,13 +32,13 @@ * {@link StaticArgumentTrait#PASS_COLUMNS_THROUGH}. */ @Internal -class ReadableInternalTimeContext implements ProcessTableFunction.TimeContext { +public class ReadableInternalTimeContext implements ProcessTableFunction.TimeContext { protected long tableWatermark; protected long currentWatermark; protected @Nullable Long time; - void setTime(long tableWatermark, long currentWatermark, @Nullable Long time) { + public void setTime(long tableWatermark, long currentWatermark, @Nullable Long time) { this.tableWatermark = tableWatermark; this.currentWatermark = currentWatermark; this.time = time; diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java index ce97c630c80c75..246edfdb87fcfe 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java @@ -35,13 +35,13 @@ /** Internal {@link TimeContext} for PTFs with set semantics. */ @Internal -class WritableInternalTimeContext extends ReadableInternalTimeContext { +public class WritableInternalTimeContext extends ReadableInternalTimeContext { private final MapState namedTimersMapState; private final InternalTimerService namedTimerService; private final InternalTimerService unnamedTimerService; - WritableInternalTimeContext( + public WritableInternalTimeContext( MapState namedTimersMapState, InternalTimerService namedTimerService, InternalTimerService unnamedTimerService) {