diff --git a/flink-python/pyflink/fn_execution/beam/beam_operations.py b/flink-python/pyflink/fn_execution/beam/beam_operations.py index 184f519ec926f..a27c413ee2f07 100644 --- a/flink-python/pyflink/fn_execution/beam/beam_operations.py +++ b/flink-python/pyflink/fn_execution/beam/beam_operations.py @@ -37,6 +37,7 @@ from pyflink.fn_execution.datastream.process import operations import pyflink.fn_execution.table.async_function.operations as async_table_operations import pyflink.fn_execution.table.operations as table_operations +import pyflink.fn_execution.table.process_table_function as process_table_function # ----------------- UDF -------------------- @@ -136,6 +137,15 @@ def create_pandas_over_window_aggregate_function( def create_data_stream_keyed_process_function(factory, transform_id, transform_proto, parameter, consumers): urn = parameter.do_fn.urn + if urn == process_table_function.PROCESS_TABLE_FUNCTION_URN: + payload = proto_utils.parse_Bytes( + parameter.do_fn.payload, + flink_fn_execution_pb2.UserDefinedProcessTableFunction) + operation_cls = beam_operations.StatefulFunctionOperation \ + if payload.HasField('key_type') else beam_operations.StatelessFunctionOperation + return _create_user_defined_function_operation( + factory, transform_proto, consumers, payload, operation_cls, + process_table_function.ProcessTableFunctionOperation) payload = proto_utils.parse_Bytes( parameter.do_fn.payload, flink_fn_execution_pb2.UserDefinedDataStreamFunction) if urn == datastream_operations.DATA_STREAM_STATELESS_FUNCTION_URN: @@ -176,11 +186,11 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers, else: operator_state_backend = None - if hasattr(serialized_fn, "key_type"): + if hasattr(serialized_fn, "key_type") and serialized_fn.HasField("key_type"): # keyed operation, need to create the KeyedStateBackend. row_schema = serialized_fn.key_type.row_schema key_row_coder = FlattenRowCoder([from_proto(f.type) for f in row_schema.fields]) - if serialized_fn.HasField('group_window'): + if hasattr(serialized_fn, 'group_window') and serialized_fn.HasField('group_window'): if serialized_fn.group_window.is_time_window: window_coder = TimeWindowCoder() else: diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py index 96984f80d5c77..6bdc9c56045f5 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py @@ -31,7 +31,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf4\x06\n\x1fUserDefinedProcessTableFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12]\n\targuments\x18\x02 \x03(\x0b\x32J.org.apache.flink.fn_execution.v1.UserDefinedProcessTableFunction.Argument\x12W\n\x06states\x18\x03 \x03(\x0b\x32G.org.apache.flink.fn_execution.v1.UserDefinedProcessTableFunction.State\x12\x44\n\x08key_type\x18\x04 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x14\n\x0chas_on_timer\x18\x05 \x01(\x08\x12\x16\n\x0emetric_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fprofile_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x0b \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0c \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x1a|\n\x08\x41rgument\x12\x0c\n\x04name\x18\x01 \x01(\t\x12@\n\x04type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x10\n\x08is_table\x18\x03 \x01(\x08\x12\x0e\n\x06traits\x18\x04 \x03(\t\x1ak\n\x05State\x12\x0c\n\x04name\x18\x01 \x01(\t\x12@\n\x04type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x12\n\nttl_millis\x18\x03 \x01(\x03\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -49,108 +49,114 @@ _globals['_ASYNCOPTIONS']._serialized_end=559 _globals['_USERDEFINEDFUNCTIONS']._serialized_start=562 _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1013 - _globals['_OVERWINDOW']._serialized_start=1016 - _globals['_OVERWINDOW']._serialized_end=1365 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1157 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1365 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1368 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2147 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1633 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2147 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=1896 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=1980 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=1983 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2134 - _globals['_GROUPWINDOW']._serialized_start=2150 - _globals['_GROUPWINDOW']._serialized_end=2706 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2514 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2605 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2607 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2706 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2709 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3420 - _globals['_SCHEMA']._serialized_start=3423 - _globals['_SCHEMA']._serialized_end=5461 - _globals['_SCHEMA_MAPINFO']._serialized_start=3498 - _globals['_SCHEMA_MAPINFO']._serialized_end=3649 - _globals['_SCHEMA_TIMEINFO']._serialized_start=3651 - _globals['_SCHEMA_TIMEINFO']._serialized_end=3680 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3682 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3716 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3718 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3762 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3764 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3803 - _globals['_SCHEMA_DECIMALINFO']._serialized_start=3805 - _globals['_SCHEMA_DECIMALINFO']._serialized_end=3852 - _globals['_SCHEMA_BINARYINFO']._serialized_start=3854 - _globals['_SCHEMA_BINARYINFO']._serialized_end=3882 - _globals['_SCHEMA_VARBINARYINFO']._serialized_start=3884 - _globals['_SCHEMA_VARBINARYINFO']._serialized_end=3915 - _globals['_SCHEMA_CHARINFO']._serialized_start=3917 - _globals['_SCHEMA_CHARINFO']._serialized_end=3943 - _globals['_SCHEMA_VARCHARINFO']._serialized_start=3945 - _globals['_SCHEMA_VARCHARINFO']._serialized_end=3974 - _globals['_SCHEMA_FIELDTYPE']._serialized_start=3977 - _globals['_SCHEMA_FIELDTYPE']._serialized_end=5049 - _globals['_SCHEMA_FIELD']._serialized_start=5051 - _globals['_SCHEMA_FIELD']._serialized_end=5159 - _globals['_SCHEMA_TYPENAME']._serialized_start=5162 - _globals['_SCHEMA_TYPENAME']._serialized_end=5461 - _globals['_TYPEINFO']._serialized_start=5464 - _globals['_TYPEINFO']._serialized_end=6811 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=5958 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6097 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6100 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6284 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6193 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6284 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6286 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6366 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6368 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6398 - _globals['_TYPEINFO_TYPENAME']._serialized_start=6401 - _globals['_TYPEINFO_TYPENAME']._serialized_end=6798 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6814 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7791 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7309 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7615 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7618 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7791 - _globals['_STATEDESCRIPTOR']._serialized_start=7794 - _globals['_STATEDESCRIPTOR']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7926 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8397 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8575 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8663 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8665 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8740 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8743 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9351 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9353 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9451 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9453 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9497 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9565 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9567 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9641 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9643 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9686 - _globals['_CODERINFODESCRIPTOR']._serialized_start=9689 - _globals['_CODERINFODESCRIPTOR']._serialized_end=10698 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10282 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10356 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10358 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10425 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10427 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10496 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10498 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10577 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10579 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10651 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10653 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10685 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION']._serialized_start=1016 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION']._serialized_end=1900 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_ARGUMENT']._serialized_start=1667 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_ARGUMENT']._serialized_end=1791 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_STATE']._serialized_start=1793 + _globals['_USERDEFINEDPROCESSTABLEFUNCTION_STATE']._serialized_end=1900 + _globals['_OVERWINDOW']._serialized_start=1903 + _globals['_OVERWINDOW']._serialized_end=2252 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=2044 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=2252 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=2255 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=3034 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=2520 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=3034 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=2783 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=2867 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=2870 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=3021 + _globals['_GROUPWINDOW']._serialized_start=3037 + _globals['_GROUPWINDOW']._serialized_end=3593 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=3401 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=3492 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=3494 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=3593 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=3596 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=4307 + _globals['_SCHEMA']._serialized_start=4310 + _globals['_SCHEMA']._serialized_end=6348 + _globals['_SCHEMA_MAPINFO']._serialized_start=4385 + _globals['_SCHEMA_MAPINFO']._serialized_end=4536 + _globals['_SCHEMA_TIMEINFO']._serialized_start=4538 + _globals['_SCHEMA_TIMEINFO']._serialized_end=4567 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=4569 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=4603 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=4605 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=4649 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=4651 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=4690 + _globals['_SCHEMA_DECIMALINFO']._serialized_start=4692 + _globals['_SCHEMA_DECIMALINFO']._serialized_end=4739 + _globals['_SCHEMA_BINARYINFO']._serialized_start=4741 + _globals['_SCHEMA_BINARYINFO']._serialized_end=4769 + _globals['_SCHEMA_VARBINARYINFO']._serialized_start=4771 + _globals['_SCHEMA_VARBINARYINFO']._serialized_end=4802 + _globals['_SCHEMA_CHARINFO']._serialized_start=4804 + _globals['_SCHEMA_CHARINFO']._serialized_end=4830 + _globals['_SCHEMA_VARCHARINFO']._serialized_start=4832 + _globals['_SCHEMA_VARCHARINFO']._serialized_end=4861 + _globals['_SCHEMA_FIELDTYPE']._serialized_start=4864 + _globals['_SCHEMA_FIELDTYPE']._serialized_end=5936 + _globals['_SCHEMA_FIELD']._serialized_start=5938 + _globals['_SCHEMA_FIELD']._serialized_end=6046 + _globals['_SCHEMA_TYPENAME']._serialized_start=6049 + _globals['_SCHEMA_TYPENAME']._serialized_end=6348 + _globals['_TYPEINFO']._serialized_start=6351 + _globals['_TYPEINFO']._serialized_end=7698 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=6845 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6984 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6987 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=7171 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=7080 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=7171 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=7173 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=7253 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=7255 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=7285 + _globals['_TYPEINFO_TYPENAME']._serialized_start=7288 + _globals['_TYPEINFO_TYPENAME']._serialized_end=7685 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=7701 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=8678 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=8196 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=8502 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=8505 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=8678 + _globals['_STATEDESCRIPTOR']._serialized_start=8681 + _globals['_STATEDESCRIPTOR']._serialized_end=10573 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=8813 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=10573 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=9284 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=10382 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=9462 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=9550 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=9552 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=9627 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=9630 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=10238 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=10240 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=10338 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=10340 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=10382 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=10384 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=10452 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=10454 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=10528 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=10530 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=10573 + _globals['_CODERINFODESCRIPTOR']._serialized_start=10576 + _globals['_CODERINFODESCRIPTOR']._serialized_end=11585 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=11169 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=11243 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=11245 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=11312 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=11314 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=11383 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=11385 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=11464 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=11466 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=11538 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=11540 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=11572 # @@protoc_insertion_point(module_scope) diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi index b6b77c6894609..f16d59bac2142 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi @@ -87,6 +87,54 @@ class UserDefinedFunctions(_message.Message): runtime_context: UserDefinedDataStreamFunction.RuntimeContext def __init__(self, udfs: _Optional[_Iterable[_Union[UserDefinedFunction, _Mapping]]] = ..., metric_enabled: bool = ..., windows: _Optional[_Iterable[_Union[OverWindow, _Mapping]]] = ..., profile_enabled: bool = ..., job_parameters: _Optional[_Iterable[_Union[JobParameter, _Mapping]]] = ..., async_options: _Optional[_Union[AsyncOptions, _Mapping]] = ..., runtime_context: _Optional[_Union[UserDefinedDataStreamFunction.RuntimeContext, _Mapping]] = ...) -> None: ... +class UserDefinedProcessTableFunction(_message.Message): + __slots__ = ("payload", "arguments", "states", "key_type", "has_on_timer", "metric_enabled", "profile_enabled", "state_cache_size", "map_state_read_cache_size", "map_state_write_cache_size", "job_parameters", "runtime_context") + class Argument(_message.Message): + __slots__ = ("name", "type", "is_table", "traits") + NAME_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + IS_TABLE_FIELD_NUMBER: _ClassVar[int] + TRAITS_FIELD_NUMBER: _ClassVar[int] + name: str + type: Schema.FieldType + is_table: bool + traits: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, name: _Optional[str] = ..., type: _Optional[_Union[Schema.FieldType, _Mapping]] = ..., is_table: bool = ..., traits: _Optional[_Iterable[str]] = ...) -> None: ... + class State(_message.Message): + __slots__ = ("name", "type", "ttl_millis") + NAME_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + TTL_MILLIS_FIELD_NUMBER: _ClassVar[int] + name: str + type: Schema.FieldType + ttl_millis: int + def __init__(self, name: _Optional[str] = ..., type: _Optional[_Union[Schema.FieldType, _Mapping]] = ..., ttl_millis: _Optional[int] = ...) -> None: ... + PAYLOAD_FIELD_NUMBER: _ClassVar[int] + ARGUMENTS_FIELD_NUMBER: _ClassVar[int] + STATES_FIELD_NUMBER: _ClassVar[int] + KEY_TYPE_FIELD_NUMBER: _ClassVar[int] + HAS_ON_TIMER_FIELD_NUMBER: _ClassVar[int] + METRIC_ENABLED_FIELD_NUMBER: _ClassVar[int] + PROFILE_ENABLED_FIELD_NUMBER: _ClassVar[int] + STATE_CACHE_SIZE_FIELD_NUMBER: _ClassVar[int] + MAP_STATE_READ_CACHE_SIZE_FIELD_NUMBER: _ClassVar[int] + MAP_STATE_WRITE_CACHE_SIZE_FIELD_NUMBER: _ClassVar[int] + JOB_PARAMETERS_FIELD_NUMBER: _ClassVar[int] + RUNTIME_CONTEXT_FIELD_NUMBER: _ClassVar[int] + payload: bytes + arguments: _containers.RepeatedCompositeFieldContainer[UserDefinedProcessTableFunction.Argument] + states: _containers.RepeatedCompositeFieldContainer[UserDefinedProcessTableFunction.State] + key_type: Schema.FieldType + has_on_timer: bool + metric_enabled: bool + profile_enabled: bool + state_cache_size: int + map_state_read_cache_size: int + map_state_write_cache_size: int + job_parameters: _containers.RepeatedCompositeFieldContainer[JobParameter] + runtime_context: UserDefinedDataStreamFunction.RuntimeContext + def __init__(self, payload: _Optional[bytes] = ..., arguments: _Optional[_Iterable[_Union[UserDefinedProcessTableFunction.Argument, _Mapping]]] = ..., states: _Optional[_Iterable[_Union[UserDefinedProcessTableFunction.State, _Mapping]]] = ..., key_type: _Optional[_Union[Schema.FieldType, _Mapping]] = ..., has_on_timer: bool = ..., metric_enabled: bool = ..., profile_enabled: bool = ..., state_cache_size: _Optional[int] = ..., map_state_read_cache_size: _Optional[int] = ..., map_state_write_cache_size: _Optional[int] = ..., job_parameters: _Optional[_Iterable[_Union[JobParameter, _Mapping]]] = ..., runtime_context: _Optional[_Union[UserDefinedDataStreamFunction.RuntimeContext, _Mapping]] = ...) -> None: ... + class OverWindow(_message.Message): __slots__ = ("window_type", "lower_boundary", "upper_boundary") class WindowType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): diff --git a/flink-python/pyflink/fn_execution/table/process_table_function.py b/flink-python/pyflink/fn_execution/table/process_table_function.py new file mode 100644 index 0000000000000..2f5e14071ee76 --- /dev/null +++ b/flink-python/pyflink/fn_execution/table/process_table_function.py @@ -0,0 +1,263 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import datetime + +from pyflink.common import Instant, Row, Time +from pyflink.datastream.state import StateTtlConfig +from pyflink.fn_execution import pickle +from pyflink.fn_execution.coders import from_proto +from pyflink.fn_execution.table.operations import BaseOperation +from pyflink.fn_execution.utils.operation_utils import normalize_table_function_result + + +PROCESS_TABLE_FUNCTION_URN = "flink:transform:process_table_function:v1" + +REGISTER_ANONYMOUS = 0 +REGISTER_NAMED = 1 +DELETE_ANONYMOUS = 2 +DELETE_NAMED = 3 +CLEAR_ALL = 4 +TRIGGER = 5 + + +def _state_key(key): + return list(key) if isinstance(key, Row) else key + + +def _to_millis(value): + if isinstance(value, bool): + raise TypeError("A timer timestamp must not be a bool.") + if isinstance(value, int): + return value + if isinstance(value, Instant): + return value.to_epoch_milli() + if isinstance(value, datetime.datetime): + if value.tzinfo is not None: + raise ValueError("Timer datetime values must be UTC-naive.") + epoch = datetime.datetime(1970, 1, 1) + return int((value - epoch).total_seconds() * 1000) + raise TypeError("Time values must be int, Instant, or UTC-naive datetime.datetime.") + + +def _from_millis(value, conversion_type): + if value is None: + return None + if conversion_type is int: + return value + if conversion_type is Instant: + return Instant.of_epoch_milli(value) + if conversion_type is datetime.datetime: + return datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=value) + raise TypeError("Time context type must be int, Instant, or datetime.datetime.") + + +class _TimerCommandEmitter(object): + + def __init__(self, keyed_state_backend): + self._keyed_state_backend = keyed_state_backend + self._timer_coder_impl = None + self._output_stream = None + + def add_timer_info(self, timer_info): + self._timer_coder_impl = timer_info.timer_coder_impl + self._output_stream = timer_info.output_stream + + def emit(self, operation, timestamp=None, name=None): + if self._timer_coder_impl is None: + raise RuntimeError("Timer services are not available for this PTF call.") + from apache_beam.transforms import userstate + from apache_beam.transforms.window import GlobalWindow + + key = self._keyed_state_backend.get_current_key() + timer_key = Row(*key) if isinstance(key, list) else key + timer_data = Row(operation, timestamp, name, timer_key, None, None) + timer = userstate.Timer( + user_key=timer_data, + dynamic_timer_tag='', + windows=(GlobalWindow(),), + clear_bit=True, + fire_timestamp=None, + hold_timestamp=None, + paneinfo=None) + self._timer_coder_impl.encode_to_stream(timer, self._output_stream, True) + self._timer_coder_impl._key_coder_impl._value_coder._output_stream.maybe_flush() + + +class _ProcessTableTimeContext(object): + + def __init__(self, context, conversion_type): + if conversion_type not in (int, Instant, datetime.datetime): + raise TypeError("Time context type must be int, Instant, or datetime.datetime.") + self._context = context + self._conversion_type = conversion_type + + def time(self): + return _from_millis(self._context._time, self._conversion_type) + + def table_watermark(self): + return _from_millis(self._context._table_watermark, self._conversion_type) + + def current_watermark(self): + return _from_millis(self._context._current_watermark, self._conversion_type) + + def register_on_time(self, *args): + if len(args) == 1: + self._context._timer_emitter.emit(REGISTER_ANONYMOUS, _to_millis(args[0])) + elif len(args) == 2 and isinstance(args[0], str): + self._context._timer_emitter.emit( + REGISTER_NAMED, _to_millis(args[1]), args[0]) + else: + raise TypeError("register_on_time expects time or name, time.") + + def clear_timer(self, time_or_name): + if isinstance(time_or_name, str): + self._context._timer_emitter.emit(DELETE_NAMED, name=time_or_name) + else: + self._context._timer_emitter.emit( + DELETE_ANONYMOUS, timestamp=_to_millis(time_or_name)) + + def clear_all_timers(self): + self._context.clear_all_timers() + + +class _ProcessTableFunctionContext(object): + + def __init__(self, timer_emitter, state_names=None): + self._timer_emitter = timer_emitter + self._state_names = None if state_names is None else frozenset(state_names) + self._cleared_states = set() + self._time = None + self._table_watermark = None + self._current_watermark = None + self._current_timer = None + + def set_event(self, time, table_watermark, current_watermark, current_timer=None): + self._cleared_states.clear() + self._time = time + self._table_watermark = table_watermark + self._current_watermark = current_watermark + self._current_timer = current_timer + + def time_context(self, conversion_type): + return _ProcessTableTimeContext(self, conversion_type) + + def clear_state(self, name): + if self._state_names is not None and name not in self._state_names: + raise ValueError("Unknown state entry: %s" % name) + self._cleared_states.add(name) + + def clear_all_state(self): + self._cleared_states.add(None) + + def clear_all_timers(self): + self._timer_emitter.emit(CLEAR_ALL) + + def clear_all(self): + self.clear_all_state() + self.clear_all_timers() + + def current_timer(self): + return self._current_timer + + +class ProcessTableFunctionOperation(BaseOperation): + + def __init__(self, serialized_fn, keyed_state_backend=None): + self.keyed_state_backend = keyed_state_backend + self._function = pickle.loads(serialized_fn.payload) + self._state_specs = list(serialized_fn.states) + self._state_handles = [] + self._timer_emitter = _TimerCommandEmitter(keyed_state_backend) + self._context = _ProcessTableFunctionContext( + self._timer_emitter, [state.name for state in self._state_specs]) + super(ProcessTableFunctionOperation, self).__init__(serialized_fn) + + def generate_func(self, serialized_fn): + return lambda value: (), [self._function] + + def open(self): + super(ProcessTableFunctionOperation, self).open() + if self.keyed_state_backend is None: + return + self._open_state_handles() + + def _open_state_handles(self): + for state in self._state_specs: + ttl_config = None + if state.ttl_millis > 0: + ttl_config = StateTtlConfig.new_builder( + Time.milliseconds(state.ttl_millis)).build() + self._state_handles.append(self.keyed_state_backend.get_value_state( + state.name, from_proto(state.type), ttl_config)) + + def finish(self): + super(ProcessTableFunctionOperation, self).finish() + if self.keyed_state_backend is not None: + self.keyed_state_backend.commit() + + def add_timer_info(self, timer_info): + self._timer_emitter.add_timer_info(timer_info) + + def process_element(self, value): + key, arguments, time, table_watermark, current_watermark = value + if self.keyed_state_backend is not None: + self.keyed_state_backend.set_current_key(_state_key(key)) + self._context.set_event(time, table_watermark, current_watermark) + return self._invoke(self._function.eval, list(arguments)) + + def process_timer(self, timer_data): + operation, timestamp, name, key, table_watermark, current_watermark = timer_data + if operation != TRIGGER: + raise ValueError("Unexpected PTF timer input operation: %s" % operation) + self.keyed_state_backend.set_current_key(_state_key(key)) + self._context.set_event(timestamp, table_watermark, current_watermark, name) + return self._invoke(self._function.on_timer, []) + + def _invoke(self, callback, arguments): + states = self._read_states() + + def invoke(): + completed = False + try: + results = callback(self._context, *states, *arguments) + for result in normalize_table_function_result(results): + yield result + completed = True + finally: + if completed: + self._write_states(states) + + return invoke() + + def _read_states(self): + states = [] + for spec, handle in zip(self._state_specs, self._state_handles): + value = handle.value() + if value is None: + value = Row(**{field.name: None for field in spec.type.row_schema.fields}) + states.append(value) + return states + + def _write_states(self, states): + clear_all = None in self._context._cleared_states + for spec, handle, value in zip(self._state_specs, self._state_handles, states): + if clear_all or spec.name in self._context._cleared_states or all( + field is None for field in value._values): + handle.clear() + else: + handle.update(value) diff --git a/flink-python/pyflink/fn_execution/tests/test_process_table_function.py b/flink-python/pyflink/fn_execution/tests/test_process_table_function.py new file mode 100644 index 0000000000000..c86657d4239d3 --- /dev/null +++ b/flink-python/pyflink/fn_execution/tests/test_process_table_function.py @@ -0,0 +1,285 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import datetime +import unittest +from types import SimpleNamespace +from unittest import mock + +from pyflink.common import Instant, Row +from pyflink.fn_execution.table.process_table_function import ( + CLEAR_ALL, + DELETE_ANONYMOUS, + DELETE_NAMED, + REGISTER_ANONYMOUS, + REGISTER_NAMED, + TRIGGER, + ProcessTableFunctionOperation, + _ProcessTableFunctionContext, +) + + +class _StateHandle(object): + + def __init__(self, value=None): + self._value = value + self.updated = [] + self.clear_count = 0 + + def value(self): + return self._value + + def update(self, value): + self.updated.append(value) + self._value = value + + def clear(self): + self.clear_count += 1 + self._value = None + + +class _TimerEmitter(object): + + def __init__(self): + self.commands = [] + + def emit(self, operation, timestamp=None, name=None): + self.commands.append((operation, timestamp, name)) + + +class _KeyedBackend(object): + + def __init__(self): + self.current_key = None + + def set_current_key(self, key): + self.current_key = key + + +class _StateBackend(_KeyedBackend): + + def __init__(self): + super(_StateBackend, self).__init__() + self.requests = [] + + def get_value_state(self, name, coder, ttl_config): + handle = _StateHandle() + self.requests.append((name, coder, ttl_config, handle)) + return handle + + +def _state_spec(name, *field_names): + fields = [SimpleNamespace(name=field_name) for field_name in field_names] + return SimpleNamespace( + name=name, + type=SimpleNamespace(row_schema=SimpleNamespace(fields=fields))) + + +def _state_spec_with_ttl(name, ttl_millis, *field_names): + spec = _state_spec(name, *field_names) + spec.ttl_millis = ttl_millis + return spec + + +def _operation(handle): + operation = object.__new__(ProcessTableFunctionOperation) + operation._state_specs = [_state_spec("memory", "count")] + operation._state_handles = [handle] + operation._context = _ProcessTableFunctionContext(_TimerEmitter()) + operation._context.set_event(1000, 900, 800) + return operation + + +class ProcessTableFunctionOperationTests(unittest.TestCase): + + def test_empty_state_is_injected_and_written_after_generator_completion(self): + handle = _StateHandle() + operation = _operation(handle) + + def callback(ctx, memory): + self.assertIsNone(memory.count) + memory["count"] = 1 + yield Row(1) + + self.assertEqual([[1]], list(operation._invoke(callback, []))) + self.assertEqual(1, handle.updated[0].count) + self.assertEqual(0, handle.clear_count) + + def test_explicit_clear_wins_over_state_mutation(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def callback(ctx, memory): + memory["count"] = 2 + ctx.clear_state("memory") + return None + + self.assertEqual([], list(operation._invoke(callback, []))) + self.assertEqual([], handle.updated) + self.assertEqual(1, handle.clear_count) + + def test_unknown_state_clear_is_rejected(self): + context = _ProcessTableFunctionContext(_TimerEmitter(), ["memory"]) + + with self.assertRaisesRegex(ValueError, "Unknown state entry: missing"): + context.clear_state("missing") + + def test_all_null_state_is_cleared(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def callback(ctx, memory): + memory["count"] = None + return None + + self.assertEqual([], list(operation._invoke(callback, []))) + self.assertEqual(1, handle.clear_count) + + def test_state_is_not_written_when_generator_raises(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def callback(ctx, memory): + memory["count"] = 2 + yield Row(2) + raise RuntimeError("boom") + + with self.assertRaisesRegex(RuntimeError, "boom"): + list(operation._invoke(callback, [])) + self.assertEqual([], handle.updated) + self.assertEqual(0, handle.clear_count) + + def test_multiple_states_follow_declaration_order(self): + first = _StateHandle(Row(count=1)) + second = _StateHandle() + operation = _operation(first) + operation._state_specs = [ + _state_spec("first", "count"), + _state_spec("second", "count"), + ] + operation._state_handles = [first, second] + + def callback(ctx, first_state, second_state): + first_state["count"] += 1 + second_state["count"] = 10 + return None + + self.assertEqual([], list(operation._invoke(callback, []))) + self.assertEqual(2, first.updated[0].count) + self.assertEqual(10, second.updated[0].count) + + def test_state_ttl_is_forwarded_to_remote_backend(self): + backend = _StateBackend() + operation = object.__new__(ProcessTableFunctionOperation) + operation.keyed_state_backend = backend + operation._state_specs = [ + _state_spec_with_ttl("expiring", 1234, "count"), + _state_spec_with_ttl("persistent", 0, "count"), + ] + operation._state_handles = [] + + with mock.patch( + 'pyflink.fn_execution.table.process_table_function.from_proto', + return_value='state-coder'): + operation._open_state_handles() + + self.assertEqual(["expiring", "persistent"], + [request[0] for request in backend.requests]) + self.assertEqual(["state-coder", "state-coder"], + [request[1] for request in backend.requests]) + self.assertEqual(1234, backend.requests[0][2].get_ttl().to_milliseconds()) + self.assertIsNone(backend.requests[1][2]) + self.assertEqual(2, len(operation._state_handles)) + + def test_zero_to_many_results(self): + handle = _StateHandle(Row(count=1)) + operation = _operation(handle) + + def no_results(ctx, memory): + return None + + def many_results(ctx, memory): + yield Row(1) + yield Row(2) + + self.assertEqual([], list(operation._invoke(no_results, []))) + self.assertEqual([[1], [2]], list(operation._invoke(many_results, []))) + + def test_process_timer_sets_key_and_current_timer(self): + handle = _StateHandle(Row(count=3)) + backend = _KeyedBackend() + operation = _operation(handle) + operation.keyed_state_backend = backend + + class Function(object): + def on_timer(self, ctx, memory): + yield Row(memory.count, ctx.current_timer()) + + operation._function = Function() + key = Row("user-1") + results = list(operation.process_timer( + (TRIGGER, 1200, "timeout", key, 1100, 1150))) + + self.assertEqual(["user-1"], backend.current_key) + self.assertEqual([[3, "timeout"]], results) + self.assertEqual(1200, operation._context.time_context(int).time()) + + +class ProcessTableFunctionTimeContextTests(unittest.TestCase): + + def setUp(self): + self.emitter = _TimerEmitter() + self.context = _ProcessTableFunctionContext(self.emitter) + self.context.set_event(1000, 900, 800, "timeout") + + def test_time_context_conversions(self): + self.assertEqual(1000, self.context.time_context(int).time()) + self.assertEqual( + 1000, self.context.time_context(Instant).time().to_epoch_milli()) + self.assertEqual( + datetime.datetime(1970, 1, 1, 0, 0, 1), + self.context.time_context(datetime.datetime).time()) + self.assertEqual("timeout", self.context.current_timer()) + with self.assertRaises(TypeError): + self.context.time_context(str) + + def test_register_and_clear_timers(self): + time_context = self.context.time_context(int) + time_context.register_on_time(1100) + time_context.register_on_time("timeout", Instant.of_epoch_milli(1200)) + time_context.clear_timer(1100) + time_context.clear_timer("timeout") + time_context.clear_all_timers() + + self.assertEqual( + [ + (REGISTER_ANONYMOUS, 1100, None), + (REGISTER_NAMED, 1200, "timeout"), + (DELETE_ANONYMOUS, 1100, None), + (DELETE_NAMED, None, "timeout"), + (CLEAR_ALL, None, None), + ], + self.emitter.commands) + + def test_rejects_timezone_aware_datetime(self): + with self.assertRaisesRegex(ValueError, "UTC-naive"): + self.context.time_context(datetime.datetime).register_on_time( + datetime.datetime.now(datetime.timezone.utc)) + + +if __name__ == '__main__': + unittest.main() diff --git a/flink-python/pyflink/proto/flink-fn-execution.proto b/flink-python/pyflink/proto/flink-fn-execution.proto index 4ab5616b011b8..9d58707a61c87 100644 --- a/flink-python/pyflink/proto/flink-fn-execution.proto +++ b/flink-python/pyflink/proto/flink-fn-execution.proto @@ -87,6 +87,35 @@ message UserDefinedFunctions { UserDefinedDataStreamFunction.RuntimeContext runtime_context = 7; } +// A Python user-defined process table function and its explicit runtime contract. +message UserDefinedProcessTableFunction { + message Argument { + string name = 1; + Schema.FieldType type = 2; + bool is_table = 3; + repeated string traits = 4; + } + + message State { + string name = 1; + Schema.FieldType type = 2; + int64 ttl_millis = 3; + } + + bytes payload = 1; + repeated Argument arguments = 2; + repeated State states = 3; + Schema.FieldType key_type = 4; + bool has_on_timer = 5; + bool metric_enabled = 6; + bool profile_enabled = 7; + int32 state_cache_size = 8; + int32 map_state_read_cache_size = 9; + int32 map_state_write_cache_size = 10; + repeated JobParameter job_parameters = 11; + UserDefinedDataStreamFunction.RuntimeContext runtime_context = 12; +} + // Used to describe the info of over window in pandas batch over window aggregation message OverWindow { enum WindowType { diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py index 6fc10c9393bf7..4ab2c057ca25f 100644 --- a/flink-python/pyflink/table/__init__.py +++ b/flink-python/pyflink/table/__init__.py @@ -40,6 +40,8 @@ Base interface for user-defined scalar function. - :class:`TableFunction` Base interface for user-defined table function. + - :class:`ProcessTableFunction` + Base interface for user-defined process table function. - :class:`AggregateFunction` Base interface for user-defined aggregate function. - :class:`TableAggregateFunction` @@ -124,8 +126,8 @@ from pyflink.table.schema import Schema from pyflink.table.sql_dialect import SqlDialect from pyflink.table.statement_set import StatementSet -from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, Table, \ - WindowGroupedTable +from pyflink.table.table import GroupWindowedTable, GroupedTable, OverWindowedTable, \ + PartitionedTable, Table, WindowGroupedTable from pyflink.table.table_config import TableConfig from pyflink.table.table_descriptor import TableDescriptor, FormatDescriptor from pyflink.table.table_environment import (TableEnvironment, StreamTableEnvironment) @@ -134,7 +136,8 @@ from pyflink.table.table_schema import TableSchema from pyflink.table.types import DataTypes, UserDefinedType, Row, RowKind from pyflink.table.udf import FunctionContext, ScalarFunction, TableFunction, AggregateFunction, \ - TableAggregateFunction, AsyncScalarFunction + TableAggregateFunction, AsyncScalarFunction, ProcessTableFunction, \ + ProcessTableFunctionArgument, ProcessTableFunctionArgumentTrait, ProcessTableFunctionState __all__ = [ 'TableEnvironment', @@ -147,8 +150,13 @@ 'GroupWindowedTable', 'OverWindowedTable', 'WindowGroupedTable', + 'PartitionedTable', 'ScalarFunction', 'TableFunction', + 'ProcessTableFunction', + 'ProcessTableFunctionArgument', + 'ProcessTableFunctionArgumentTrait', + 'ProcessTableFunctionState', 'AggregateFunction', 'TableAggregateFunction', 'AsyncScalarFunction', diff --git a/flink-python/pyflink/table/table.py b/flink-python/pyflink/table/table.py index e7898e8752d1f..b431d162556d1 100644 --- a/flink-python/pyflink/table/table.py +++ b/flink-python/pyflink/table/table.py @@ -43,7 +43,8 @@ from pyflink.util.java_utils import to_jarray from pyflink.util.java_utils import to_j_explain_detail_arr -__all__ = ['Table', 'GroupedTable', 'GroupWindowedTable', 'OverWindowedTable', 'WindowGroupedTable'] +__all__ = ['Table', 'PartitionedTable', 'GroupedTable', 'GroupWindowedTable', + 'OverWindowedTable', 'WindowGroupedTable'] @PublicEvolving() @@ -156,6 +157,46 @@ def alias(self, field: str, *fields: str) -> 'Table': extra_fields = to_jarray(gateway.jvm.String, fields) return Table(get_method(self._j_table, "as")(field, extra_fields), self._t_env) + def partition_by(self, *fields: Expression) -> 'PartitionedTable': + """ + Partitions this table for a subsequent process table function call. + + :param fields: Partitioning expressions. + :return: A partitioned table for configuring and invoking a process table function. + + .. versionadded:: 2.4.0 + """ + return PartitionedTable( + self._j_table.partitionBy(to_expression_jarray(fields)), self._t_env) + + def as_argument(self, name: str) -> Expression: + """ + Converts this table into a named table argument for a process table function call. + + :param name: Name of the table argument declared by the function. + :return: A named table argument. + + .. versionadded:: 2.4.0 + """ + return Expression(self._j_table.asArgument(name)) + + def process(self, path: str, *arguments: Expression) -> 'Table': + """ + Calls a registered process table function with this table as its first table argument. + + :param path: Path of the registered process table function. + :param arguments: Additional named arguments of the function. + :return: The result table. + + .. versionadded:: 2.4.0 + """ + if not isinstance(path, str): + raise TypeError("Process table functions must be called by a registered name.") + gateway = get_gateway() + j_arguments = to_jarray( + gateway.jvm.Object, [_get_java_expression(argument) for argument in arguments]) + return Table(self._j_table.process(path, j_arguments), self._t_env) + def filter(self, predicate: Expression[bool]) -> 'Table': """ Filters out elements that don't pass the filter predicate. Similar to a SQL WHERE @@ -1286,6 +1327,53 @@ def from_changelog(self, *arguments: Expression) -> 'Table': return Table(self._j_table.fromChangelog(to_expression_jarray(arguments)), self._t_env) +@PublicEvolving() +class PartitionedTable(object): + """ + A table argument with partitioning and optional ordering metadata for process table functions. + + .. versionadded:: 2.4.0 + """ + + def __init__(self, j_partitioned_table, t_env): + self._j_partitioned_table = j_partitioned_table + self._t_env = t_env + + def order_by(self, *fields: Expression) -> 'PartitionedTable': + """ + Defines the ordering of rows within each partition. + + :param fields: Ordering expressions. + :return: A partitioned table with ordering metadata. + """ + return PartitionedTable( + self._j_partitioned_table.orderBy(to_expression_jarray(fields)), self._t_env) + + def as_argument(self, name: str) -> Expression: + """ + Converts this partitioned table into a named table argument. + + :param name: Name of the table argument declared by the function. + :return: A named table argument. + """ + return Expression(self._j_partitioned_table.asArgument(name)) + + def process(self, path: str, *arguments: Expression) -> Table: + """ + Calls a registered process table function with this partitioned table. + + :param path: Path of the registered process table function. + :param arguments: Additional named arguments of the function. + :return: The result table. + """ + if not isinstance(path, str): + raise TypeError("Process table functions must be called by a registered name.") + gateway = get_gateway() + j_arguments = to_jarray( + gateway.jvm.Object, [_get_java_expression(argument) for argument in arguments]) + return Table(self._j_partitioned_table.process(path, j_arguments), self._t_env) + + @PublicEvolving() class GroupedTable(object): """ diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py index f0e4bedba174e..0c66a64d684fc 100644 --- a/flink-python/pyflink/table/table_environment.py +++ b/flink-python/pyflink/table/table_environment.py @@ -36,6 +36,7 @@ from pyflink.serializers import BatchedSerializer, PickleSerializer from pyflink.table import Table, EnvironmentSettings, Expression, ExplainDetail, \ Module, ModuleEntry, Schema, ChangelogMode +from pyflink.table.expression import _get_java_expression from pyflink.table.catalog import Catalog, CatalogDescriptor from pyflink.table.model_descriptor import ModelDescriptor from pyflink.table.compiled_plan import CompiledPlan @@ -528,6 +529,23 @@ def from_path(self, path: str) -> Table: """ return Table(get_method(self._j_tenv, "from")(path), self) + def from_call(self, path: str, *arguments: Expression) -> Table: + """ + Creates a table from a registered process table function call. + + :param path: Path of the registered process table function. + :param arguments: Named arguments of the function. + :return: The result table. + + .. versionadded:: 2.4.0 + """ + if not isinstance(path, str): + raise TypeError("Process table functions must be called by a registered name.") + gateway = get_gateway() + j_arguments = to_jarray( + gateway.jvm.Object, [_get_java_expression(argument) for argument in arguments]) + return Table(self._j_tenv.fromCall(path, j_arguments), self) + def from_descriptor(self, descriptor: TableDescriptor) -> Table: """ Returns a Table backed by the given TableDescriptor. diff --git a/flink-python/pyflink/table/tests/test_environment_completeness.py b/flink-python/pyflink/table/tests/test_environment_completeness.py index ab9c69a0e0ae1..02c6ae4d8ffe4 100644 --- a/flink-python/pyflink/table/tests/test_environment_completeness.py +++ b/flink-python/pyflink/table/tests/test_environment_completeness.py @@ -40,7 +40,6 @@ def excluded_methods(cls): return { 'getCompletionHints', 'fromValues', - 'fromCall', 'fromModel', # See FLINK-25986 'loadPlan', diff --git a/flink-python/pyflink/table/tests/test_process_table_function_api.py b/flink-python/pyflink/table/tests/test_process_table_function_api.py new file mode 100644 index 0000000000000..150c42fbe458b --- /dev/null +++ b/flink-python/pyflink/table/tests/test_process_table_function_api.py @@ -0,0 +1,94 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import datetime + +from pyflink.common import Row +from pyflink.table import DataTypes, EnvironmentSettings, PartitionedTable, TableEnvironment +from pyflink.table.expressions import col, lit +from pyflink.testing.test_case_utils import PyFlinkTestCase + + +class ProcessTableFunctionAPITests(PyFlinkTestCase): + + @staticmethod + def _create_table_environment(): + table_env = TableEnvironment.create(EnvironmentSettings.in_streaming_mode()) + function_prefix = ( + "org.apache.flink.table.runtime.operators.python.process." + "TestJavaProcessTableFunctions$") + table_env.create_java_temporary_system_function( + "java_row_ptf", function_prefix + "RowSemanticFunction") + table_env.create_java_temporary_system_function( + "java_multi_ptf", function_prefix + "MultiInputFunction") + return table_env + + @staticmethod + def _create_input(table_env): + return table_env.from_elements( + [("Alice", 1, datetime.datetime(2026, 1, 1, 0, 0, 0))], + DataTypes.ROW([ + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("score", DataTypes.INT()), + DataTypes.FIELD("ts", DataTypes.TIMESTAMP(3)), + ])) + + def test_table_process_calls_registered_java_function(self): + table_env = self._create_table_environment() + orders = self._create_input(table_env) + + result = orders.process("java_row_ptf", lit(42).as_argument("increment")) + + self.assertEqual(["out"], result.get_schema().get_field_names()) + self.assertIn("ProcessTableFunction", result.explain()) + with result.execute().collect() as rows: + self.assertEqual([Row("Alice:42")], list(rows)) + + def test_from_call_accepts_multiple_partitioned_table_arguments(self): + table_env = self._create_table_environment() + orders = self._create_input(table_env) + profiles = self._create_input(table_env) + + partitioned_orders = orders.partition_by(col("name")) + self.assertIsInstance(partitioned_orders, PartitionedTable) + ordered_orders = partitioned_orders.order_by(col("ts")) + self.assertIsInstance(ordered_orders, PartitionedTable) + result = table_env.from_call( + "java_multi_ptf", + partitioned_orders.as_argument("in1"), + profiles.partition_by(col("name")).as_argument("in2"), + ) + + self.assertEqual(["name", "name0", "out"], result.get_schema().get_field_names()) + self.assertIn("ProcessTableFunction", result.explain()) + + def test_process_requires_registered_function_name(self): + table_env = self._create_table_environment() + orders = self._create_input(table_env) + + with self.assertRaisesRegex(TypeError, "registered name"): + orders.process(object()) + with self.assertRaisesRegex(TypeError, "registered name"): + orders.partition_by(col("name")).process(object()) + with self.assertRaisesRegex(TypeError, "registered name"): + table_env.from_call(object(), orders.as_argument("in1")) + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/flink-python/pyflink/table/tests/test_table_completeness.py b/flink-python/pyflink/table/tests/test_table_completeness.py index feca7e63b1a90..311c027122285 100644 --- a/flink-python/pyflink/table/tests/test_table_completeness.py +++ b/flink-python/pyflink/table/tests/test_table_completeness.py @@ -17,7 +17,7 @@ ################################################################################ from pyflink.testing.test_case_utils import PythonAPICompletenessTestCase, PyFlinkTestCase -from pyflink.table import Table +from pyflink.table import PartitionedTable, Table class TableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): @@ -39,9 +39,6 @@ def excluded_methods(cls): return { 'createTemporalTableFunction', 'getQueryOperation', - 'asArgument', - 'process', - 'partitionBy', } @classmethod @@ -56,6 +53,28 @@ def java_method_name(cls, python_method_name): return {'alias': 'as'}.get(python_method_name, python_method_name) +class PartitionedTableAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): + """ + Tests whether the Python :class:`PartitionedTable` is consistent with + Java `org.apache.flink.table.api.PartitionedTable`. + """ + + @classmethod + def python_class(cls): + return PartitionedTable + + @classmethod + def java_class(cls): + return "org.apache.flink.table.api.PartitionedTable" + + @classmethod + def excluded_methods(cls): + return { + 'fromChangelog', + 'toChangelog', + } + + if __name__ == '__main__': import unittest diff --git a/flink-python/pyflink/table/tests/test_table_environment_completeness.py b/flink-python/pyflink/table/tests/test_table_environment_completeness.py index 2e0cc5c2b390c..98f1d47bf04a1 100644 --- a/flink-python/pyflink/table/tests/test_table_environment_completeness.py +++ b/flink-python/pyflink/table/tests/test_table_environment_completeness.py @@ -43,7 +43,6 @@ def excluded_methods(cls): "registerTable", "from", "registerFunction", - "fromCall", "fromModel", } diff --git a/flink-python/pyflink/table/tests/test_udptf.py b/flink-python/pyflink/table/tests/test_udptf.py new file mode 100644 index 0000000000000..9d0526fd3d5b7 --- /dev/null +++ b/flink-python/pyflink/table/tests/test_udptf.py @@ -0,0 +1,241 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +from pyflink.common import Duration, Row +from pyflink.table import DataTypes, EnvironmentSettings, TableEnvironment +from pyflink.table.udf import ( + ProcessTableFunction, + ProcessTableFunctionArgument, + ProcessTableFunctionArgumentTrait as Trait, + ProcessTableFunctionState, + udptf, +) +from pyflink.testing.test_case_utils import PyFlinkTestCase + + +class Tokenize(ProcessTableFunction): + + def eval(self, ctx, event, separator): + for token in event.text.split(separator): + yield Row(token) + + +class CountWithTimeout(ProcessTableFunction): + + def eval(self, ctx, memory, event): + memory["count"] = (memory.count or 0) + 1 + yield Row(memory.count) + + def on_timer(self, ctx, memory): + yield Row(memory.count) + + +class UdptfTests(PyFlinkTestCase): + + @staticmethod + def _result_type(): + return DataTypes.ROW([DataTypes.FIELD("result", DataTypes.STRING())]) + + @staticmethod + def _memory_state(): + return ProcessTableFunctionState.value( + "memory", + DataTypes.ROW([DataTypes.FIELD("count", DataTypes.BIGINT())]), + ttl=Duration.of_days(1), + ) + + def test_creates_stateless_function_with_ordered_arguments(self): + function = udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + java_function = function._java_user_defined_function() + arguments = java_function.getTypeInference(None).getStaticArguments().get() + + self.assertEqual("PythonProcessTableFunction", java_function.getClass().getSimpleName()) + self.assertEqual(["event", "separator"], [argument.getName() for argument in arguments]) + self.assertFalse(java_function.hasOnTimer()) + + table_env = TableEnvironment.create(EnvironmentSettings.in_streaming_mode()) + table_env.create_temporary_system_function("tokenize", function) + self.assertIn("tokenize", table_env.list_user_defined_functions()) + + def test_creates_stateful_timer_function(self): + function = udptf( + CountWithTimeout(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", traits={Trait.SET_SEMANTIC_TABLE}), + ], + states=[self._memory_state()], + result_type=DataTypes.ROW([DataTypes.FIELD("count", DataTypes.BIGINT())]), + ) + + java_function = function._java_user_defined_function() + + self.assertTrue(java_function.hasOnTimer()) + self.assertEqual( + ["memory"], + list(java_function.getTypeInference(None).getStateTypeStrategies().keySet()), + ) + self.assertEqual( + 86_400_000, + java_function.getStateTimeToLive()[0].toMillis(), + ) + + def test_rejects_invalid_callback_signatures(self): + class InvalidEval(ProcessTableFunction): + def eval(self, ctx, separator, event): + yield Row(separator) + + class InvalidTimer(ProcessTableFunction): + def eval(self, ctx, memory, event): + yield Row(memory.count) + + def on_timer(self, memory, ctx): + yield Row(memory.count) + + with self.assertRaisesRegex(ValueError, r"Invalid eval\(\) signature"): + udptf( + InvalidEval(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + with self.assertRaisesRegex(ValueError, r"Invalid on_timer\(\) signature"): + udptf( + InvalidTimer(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", traits={Trait.SET_SEMANTIC_TABLE}), + ], + states=[self._memory_state()], + result_type=self._result_type(), + ) + + def test_requires_exactly_one_table_argument(self): + with self.assertRaisesRegex(ValueError, "exactly one table argument"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.scalar("event", DataTypes.STRING()), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + with self.assertRaisesRegex(ValueError, "exactly one table argument"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.table("separator"), + ], + result_type=self._result_type(), + ) + + def test_state_and_timer_require_set_semantics(self): + with self.assertRaisesRegex(ValueError, "State requires"): + udptf( + CountWithTimeout(), + arguments=[ProcessTableFunctionArgument.table("event")], + states=[self._memory_state()], + result_type=self._result_type(), + ) + + class StatelessTimer(ProcessTableFunction): + def eval(self, ctx, event): + yield Row(event) + + def on_timer(self, ctx): + yield Row("timer") + + with self.assertRaisesRegex(ValueError, "Timers require"): + udptf( + StatelessTimer(), + arguments=[ProcessTableFunctionArgument.table("event")], + result_type=self._result_type(), + ) + + def test_rejects_pass_through_with_timer(self): + with self.assertRaisesRegex(ValueError, "pass-through"): + udptf( + CountWithTimeout(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", + traits={ + Trait.SET_SEMANTIC_TABLE, + Trait.PASS_COLUMNS_THROUGH, + }, + ), + ], + states=[self._memory_state()], + result_type=self._result_type(), + ) + + def test_rejects_update_traits_in_first_version(self): + for update_trait in ( + Trait.SUPPORT_UPDATES, + Trait.REQUIRE_UPDATE_BEFORE, + Trait.REQUIRE_FULL_DELETE): + with self.subTest(update_trait=update_trait): + with self.assertRaisesRegex(ValueError, "do not support updating inputs"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table( + "event", + traits={Trait.SET_SEMANTIC_TABLE, update_trait}, + ), + ProcessTableFunctionArgument.scalar( + "separator", DataTypes.STRING()), + ], + result_type=self._result_type(), + ) + + def test_validates_argument_state_and_result_types(self): + with self.assertRaisesRegex(ValueError, "both row and set semantics"): + ProcessTableFunctionArgument.table( + "event", + traits={Trait.ROW_SEMANTIC_TABLE, Trait.SET_SEMANTIC_TABLE}, + ) + with self.assertRaisesRegex(TypeError, "state must use a ROW"): + ProcessTableFunctionState.value("memory", DataTypes.BIGINT()) + with self.assertRaisesRegex(TypeError, "result_type must be a ROW"): + udptf( + Tokenize(), + arguments=[ + ProcessTableFunctionArgument.table("event"), + ProcessTableFunctionArgument.scalar("separator", DataTypes.STRING()), + ], + result_type=DataTypes.STRING(), + ) + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/flink-python/pyflink/table/udf.py b/flink-python/pyflink/table/udf.py index 92bd180d1899e..3f06f93cb5119 100644 --- a/flink-python/pyflink/table/udf.py +++ b/flink-python/pyflink/table/udf.py @@ -16,9 +16,21 @@ # limitations under the License. ################################################################################ import abc +import enum import functools import inspect -from typing import Union, List, Type, Callable, TypeVar, Generic, Iterable +from typing import ( + Union, + List, + Type, + Callable, + TypeVar, + Generic, + Iterable, + Optional, + Sequence, + Set, +) from pyflink.java_gateway import get_gateway from pyflink.metrics import MetricGroup @@ -28,7 +40,9 @@ from pyflink.util.api_stability_decorators import PublicEvolving, Internal __all__ = ['FunctionContext', 'AggregateFunction', 'ScalarFunction', 'TableFunction', - 'TableAggregateFunction', 'AsyncScalarFunction', 'udf', 'udtf', 'udaf', 'udtaf'] + 'TableAggregateFunction', 'AsyncScalarFunction', 'ProcessTableFunction', + 'ProcessTableFunctionArgument', 'ProcessTableFunctionState', + 'ProcessTableFunctionArgumentTrait', 'udf', 'udtf', 'udptf', 'udaf', 'udtaf'] @PublicEvolving() @@ -232,6 +246,129 @@ def eval(self, *args): pass +@PublicEvolving() +class ProcessTableFunction(UserDefinedFunction): + """ + Base interface for a Python user-defined process table function. + + A process table function consumes table and scalar arguments and can emit zero, one, or + multiple rows. Stateful functions receive their declared state entries as mutable + :class:`~pyflink.common.Row` objects before the call arguments. + + .. versionadded:: 2.4.0 + """ + + @abc.abstractmethod + def eval(self, ctx, *args): + """ + Processes an input row and yields zero, one, or multiple result rows. + """ + pass + + def on_timer(self, ctx, *states): + """ + Processes a firing event-time timer and yields zero, one, or multiple result rows. + """ + return () + + +@PublicEvolving() +class ProcessTableFunctionArgumentTrait(enum.Enum): + """ + Traits that describe the semantics of a process table function table argument. + + .. versionadded:: 2.4.0 + """ + + ROW_SEMANTIC_TABLE = 'ROW_SEMANTIC_TABLE' + SET_SEMANTIC_TABLE = 'SET_SEMANTIC_TABLE' + PASS_COLUMNS_THROUGH = 'PASS_COLUMNS_THROUGH' + SUPPORT_UPDATES = 'SUPPORT_UPDATES' + REQUIRE_ON_TIME = 'REQUIRE_ON_TIME' + OPTIONAL_PARTITION_BY = 'OPTIONAL_PARTITION_BY' + REQUIRE_UPDATE_BEFORE = 'REQUIRE_UPDATE_BEFORE' + REQUIRE_FULL_DELETE = 'REQUIRE_FULL_DELETE' + + +@PublicEvolving() +class ProcessTableFunctionArgument(object): + """ + Declaration of a scalar or table argument of a process table function. + + .. versionadded:: 2.4.0 + """ + + def __init__(self, name: str, data_type: Optional[DataType], is_table: bool, + traits: Set[ProcessTableFunctionArgumentTrait]): + if not isinstance(name, str) or not name: + raise ValueError("The argument name must be a non-empty string.") + self.name = name + self.data_type = data_type + self.is_table = is_table + self.traits = frozenset(traits) + + @staticmethod + def scalar(name: str, data_type: DataType) -> 'ProcessTableFunctionArgument': + """ + Declares a scalar argument with an explicit data type. + """ + if not isinstance(data_type, DataType): + raise TypeError("A scalar argument data type must be a DataType.") + return ProcessTableFunctionArgument(name, data_type, False, set()) + + @staticmethod + def table(name: str, + traits: Optional[Set[ProcessTableFunctionArgumentTrait]] = None, + data_type: Optional[DataType] = None) -> 'ProcessTableFunctionArgument': + """ + Declares a polymorphic or explicitly typed table argument. + """ + if data_type is not None and not isinstance(data_type, DataType): + raise TypeError("A table argument data type must be a DataType.") + normalized_traits = set( + traits or {ProcessTableFunctionArgumentTrait.ROW_SEMANTIC_TABLE}) + if not all(isinstance(t, ProcessTableFunctionArgumentTrait) for t in normalized_traits): + raise TypeError( + "Table argument traits must be ProcessTableFunctionArgumentTrait values.") + row_semantics = ProcessTableFunctionArgumentTrait.ROW_SEMANTIC_TABLE + set_semantics = ProcessTableFunctionArgumentTrait.SET_SEMANTIC_TABLE + if row_semantics in normalized_traits and set_semantics in normalized_traits: + raise ValueError("A table argument cannot have both row and set semantics.") + if row_semantics not in normalized_traits and set_semantics not in normalized_traits: + normalized_traits.add(row_semantics) + return ProcessTableFunctionArgument(name, data_type, True, normalized_traits) + + +@PublicEvolving() +class ProcessTableFunctionState(object): + """ + Declaration of a keyed value state entry of a process table function. + + .. versionadded:: 2.4.0 + """ + + def __init__(self, name: str, data_type: DataType, ttl=None): + if not isinstance(name, str) or not name: + raise ValueError("The state name must be a non-empty string.") + from pyflink.table.types import RowType + if not isinstance(data_type, RowType): + raise TypeError("Process table function state must use a ROW data type.") + if ttl is not None: + from pyflink.common import Duration + if not isinstance(ttl, Duration): + raise TypeError("State TTL must be a pyflink.common.Duration.") + self.name = name + self.data_type = data_type + self.ttl = ttl + + @staticmethod + def value(name: str, data_type: DataType, ttl=None) -> 'ProcessTableFunctionState': + """ + Declares a ROW value state entry. + """ + return ProcessTableFunctionState(name, data_type, ttl) + + T = TypeVar('T') ACC = TypeVar('ACC') @@ -676,6 +813,65 @@ def _create_delegate_function(self) -> UserDefinedFunction: return DelegationTableFunction(self._func) +class UserDefinedProcessTableFunctionWrapper(UserDefinedFunctionWrapper): + """ + Wrapper for a Python user-defined process table function. + """ + + def __init__(self, func: ProcessTableFunction, + arguments: Sequence[ProcessTableFunctionArgument], + states: Sequence[ProcessTableFunctionState], result_type: DataType, + deterministic=None, name=None): + super(UserDefinedProcessTableFunctionWrapper, self).__init__( + func, None, "general", deterministic, name) + self._arguments = tuple(arguments) + self._states = tuple(states) + self._result_type = result_type + self._has_on_timer = func.__class__.on_timer is not ProcessTableFunction.on_timer + + def _create_judf(self, serialized_func, j_input_types, j_function_kind): + gateway = get_gateway() + argument_names = java_utils.to_jarray( + gateway.jvm.String, [argument.name for argument in self._arguments]) + argument_types = java_utils.to_jarray( + gateway.jvm.DataType, + [_to_java_data_type(argument.data_type) if argument.data_type is not None else None + for argument in self._arguments]) + table_arguments = java_utils.to_jarray( + gateway.jvm.boolean, [argument.is_table for argument in self._arguments]) + argument_traits = java_utils.to_jarray( + gateway.jvm.String, + [','.join(sorted(trait.value for trait in argument.traits)) + for argument in self._arguments]) + state_names = java_utils.to_jarray( + gateway.jvm.String, [state.name for state in self._states]) + state_types = java_utils.to_jarray( + gateway.jvm.DataType, [_to_java_data_type(state.data_type) for state in self._states]) + state_ttls = java_utils.to_jarray( + gateway.jvm.java.time.Duration, + [state.ttl._j_duration if state.ttl is not None else None for state in self._states]) + + PythonProcessTableFunction = gateway.jvm \ + .org.apache.flink.table.functions.python.PythonProcessTableFunction + return PythonProcessTableFunction( + self._name, + bytearray(serialized_func), + argument_names, + argument_types, + table_arguments, + argument_traits, + state_names, + state_types, + state_ttls, + _to_java_data_type(self._result_type), + self._deterministic, + self._has_on_timer, + _get_python_env()) + + def _create_delegate_function(self) -> UserDefinedFunction: + raise TypeError("A process table function must extend ProcessTableFunction.") + + class UserDefinedAggregateFunctionWrapper(UserDefinedFunctionWrapper): """ Wrapper for Python user-defined aggregate function or user-defined table aggregate function. @@ -776,6 +972,11 @@ def _create_udtf(f, input_types, result_types, deterministic, name): return UserDefinedTableFunctionWrapper(f, input_types, result_types, deterministic, name) +def _create_udptf(f, arguments, states, result_type, deterministic, name): + return UserDefinedProcessTableFunctionWrapper( + f, arguments, states, result_type, deterministic, name) + + def _create_udaf(f, input_types, result_type, accumulator_type, func_type, deterministic, name): return UserDefinedAggregateFunctionWrapper( f, input_types, result_type, accumulator_type, func_type, deterministic, name) @@ -911,6 +1112,106 @@ def udtf(f: Union[Callable, TableFunction, Type] = None, return _create_udtf(f, input_types, result_types, deterministic, name) +def _validate_process_table_function_signature(func, method_name, expected_names): + method = getattr(func, method_name) + parameters = list(inspect.signature(method).parameters.values()) + supported_kinds = ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + invalid_kind = any(parameter.kind not in supported_kinds for parameter in parameters) + actual_names = [parameter.name for parameter in parameters] + if invalid_kind or actual_names != expected_names: + raise ValueError( + "Invalid {}() signature. Expected ({}) but found ({}).".format( + method_name, ', '.join(expected_names), ', '.join(actual_names))) + + +def udptf(f: Union[ProcessTableFunction, Type] = None, + arguments: Sequence[ProcessTableFunctionArgument] = None, + result_type: DataType = None, + states: Sequence[ProcessTableFunctionState] = None, + deterministic: bool = None, + name: str = None): + """ + Creates a Python user-defined process table function. + + The ``eval`` callback receives ``ctx``, all declared states, and all declared arguments in + exactly that order. An optional ``on_timer`` callback receives ``ctx`` and all states. + + :param f: process table function instance or class. + :param arguments: ordered scalar and table argument declarations. + :param result_type: ROW data type produced by the function. + :param states: optional ordered value state declarations. + :param deterministic: whether the function always returns the same result for the same input. + :param name: the function name. + :return: UserDefinedProcessTableFunctionWrapper or function. + + .. versionadded:: 2.4.0 + """ + if f is None: + return functools.partial( + udptf, + arguments=arguments, + result_type=result_type, + states=states, + deterministic=deterministic, + name=name) + + if inspect.isclass(f) and issubclass(f, ProcessTableFunction): + f = f() + if not isinstance(f, ProcessTableFunction): + raise TypeError("A process table function must extend ProcessTableFunction.") + if arguments is None: + raise ValueError("Process table function arguments must be declared.") + if not isinstance(arguments, (list, tuple)) or not all( + isinstance(argument, ProcessTableFunctionArgument) for argument in arguments): + raise TypeError("arguments must be an ordered list of ProcessTableFunctionArgument values.") + if states is None: + states = [] + if not isinstance(states, (list, tuple)) or not all( + isinstance(state, ProcessTableFunctionState) for state in states): + raise TypeError("states must be an ordered list of ProcessTableFunctionState values.") + + from pyflink.table.types import RowType + if not isinstance(result_type, RowType): + raise TypeError("result_type must be a ROW DataType.") + + table_arguments = [argument for argument in arguments if argument.is_table] + if len(table_arguments) != 1: + raise ValueError("A Python process table function must declare exactly one table argument.") + + names = [state.name for state in states] + [argument.name for argument in arguments] + if len(names) != len(set(names)): + raise ValueError("Process table function state and argument names must be unique.") + + table_argument = table_arguments[0] + unsupported_update_traits = { + ProcessTableFunctionArgumentTrait.SUPPORT_UPDATES, + ProcessTableFunctionArgumentTrait.REQUIRE_UPDATE_BEFORE, + ProcessTableFunctionArgumentTrait.REQUIRE_FULL_DELETE, + } + if table_argument.traits.intersection(unsupported_update_traits): + raise ValueError("Python process table functions do not support updating inputs.") + + set_semantics = ProcessTableFunctionArgumentTrait.SET_SEMANTIC_TABLE + if states and set_semantics not in table_argument.traits: + raise ValueError("State requires a table argument with set semantics.") + + state_names = [state.name for state in states] + argument_names = [argument.name for argument in arguments] + _validate_process_table_function_signature( + f, 'eval', ['ctx'] + state_names + argument_names) + if f.__class__.on_timer is not ProcessTableFunction.on_timer: + if set_semantics not in table_argument.traits: + raise ValueError("Timers require a table argument with set semantics.") + if ProcessTableFunctionArgumentTrait.PASS_COLUMNS_THROUGH in table_argument.traits: + raise ValueError("Timers do not support pass-through columns.") + _validate_process_table_function_signature(f, 'on_timer', ['ctx'] + state_names) + + return _create_udptf(f, arguments, states, result_type, deterministic, name) + + def udaf(f: Union[Callable, AggregateFunction, Type] = None, input_types: Union[List[DataType], DataType, str, List[str]] = None, result_type: Union[DataType, str] = None, accumulator_type: Union[DataType, str] = None, diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java index dde1558b362be..04634eb2c005a 100644 --- a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistration.java @@ -35,7 +35,7 @@ /** Handles the interaction with the Python worker for registering and deleting timers. */ @Internal -public final class TimerRegistration { +public final class TimerRegistration implements TimerRegistrationHandler { private final KeyedStateBackend keyedStateBackend; private final InternalTimerService internalTimerService; @@ -61,6 +61,7 @@ public TimerRegistration( this.baisWrapper = new DataInputViewStreamWrapper(bais); } + @Override public void setTimer(byte[] serializedTimerData) { try { bais.setBuffer(serializedTimerData, 0, serializedTimerData.length); diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java index 609f550067166..3bb5335a2b5c3 100644 --- a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationAction.java @@ -23,7 +23,7 @@ /** {@link TimerRegistrationAction} used to register Timer. */ public class TimerRegistrationAction { - private final TimerRegistration timerRegistration; + private final TimerRegistrationHandler timerRegistration; private final byte[] serializedTimerData; @@ -32,7 +32,7 @@ public class TimerRegistrationAction { private final List containingList; public TimerRegistrationAction( - TimerRegistration timerRegistration, + TimerRegistrationHandler timerRegistration, byte[] serializedTimerData, List containingList) { this.timerRegistration = timerRegistration; diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationHandler.java b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationHandler.java new file mode 100644 index 0000000000000..ce2721480445f --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/process/timer/TimerRegistrationHandler.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.operators.python.process.timer; + +import org.apache.flink.annotation.Internal; + +/** Handles timer registration commands emitted by a Python worker. */ +@Internal +public interface TimerRegistrationHandler { + + void setTimer(byte[] serializedTimerData); +} diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java index 8e5e5fdd58682..dabd3dc951b5c 100644 --- a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java @@ -36,8 +36,8 @@ import org.apache.flink.runtime.memory.OpaqueMemoryResource; import org.apache.flink.runtime.state.KeyedStateBackend; import org.apache.flink.runtime.state.OperatorStateBackend; -import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistration; import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationAction; +import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationHandler; import org.apache.flink.streaming.api.runners.python.beam.state.BeamStateRequestHandler; import org.apache.flink.util.Preconditions; import org.apache.flink.util.ShutdownHookUtil; @@ -136,7 +136,7 @@ public abstract class BeamPythonFunctionRunner implements PythonFunctionRunner { @Nullable private final TypeSerializer namespaceSerializer; - @Nullable private final TimerRegistration timerRegistration; + @Nullable private final TimerRegistrationHandler timerRegistration; private final MemoryManager memoryManager; @@ -209,7 +209,7 @@ public BeamPythonFunctionRunner( @Nullable OperatorStateBackend operatorStateBackend, @Nullable TypeSerializer keySerializer, @Nullable TypeSerializer namespaceSerializer, - @Nullable TimerRegistration timerRegistration, + @Nullable TimerRegistrationHandler timerRegistration, MemoryManager memoryManager, double managedMemoryFraction, FlinkFnApi.CoderInfoDescriptor inputCoderDescriptor, diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistration.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistration.java new file mode 100644 index 0000000000000..c8948ac73c774 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistration.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.runtime.state.KeyedStateBackend; +import org.apache.flink.streaming.api.operators.KeyContext; +import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationHandler; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.runtime.operators.process.WritableInternalTimeContext; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.util.FlinkRuntimeException; + +import static org.apache.flink.streaming.api.utils.PythonOperatorUtils.setCurrentKeyForStreaming; + +/** Applies process table function timer commands in the operator mailbox thread. */ +@Internal +final class ProcessTableTimerRegistration implements TimerRegistrationHandler { + + static final byte REGISTER_ANONYMOUS = 0; + static final byte REGISTER_NAMED = 1; + static final byte DELETE_ANONYMOUS = 2; + static final byte DELETE_NAMED = 3; + static final byte CLEAR_ALL = 4; + static final byte TRIGGER = 5; + + private final KeyContext keyContext; + private final KeyedStateBackend keyedStateBackend; + private final WritableInternalTimeContext timeContext; + private final TypeSerializer timerDataSerializer; + private final int keyArity; + private final ByteArrayInputStreamWithPos inputStream; + private final DataInputViewStreamWrapper inputView; + + ProcessTableTimerRegistration( + KeyContext keyContext, + KeyedStateBackend keyedStateBackend, + WritableInternalTimeContext timeContext, + TypeSerializer timerDataSerializer, + int keyArity) { + this.keyContext = keyContext; + this.keyedStateBackend = keyedStateBackend; + this.timeContext = timeContext; + this.timerDataSerializer = timerDataSerializer; + this.keyArity = keyArity; + this.inputStream = new ByteArrayInputStreamWithPos(); + this.inputView = new DataInputViewStreamWrapper(inputStream); + } + + @Override + public void setTimer(byte[] serializedTimerData) { + try { + inputStream.setBuffer(serializedTimerData, 0, serializedTimerData.length); + final RowData command = timerDataSerializer.deserialize(inputView); + final byte operation = command.getByte(0); + final RowData key = toBackendKey(command.getRow(3, keyArity)); + synchronized (keyedStateBackend) { + setBackendCurrentKey(key); + keyContext.setCurrentKey(key); + switch (operation) { + case REGISTER_ANONYMOUS: + timeContext.registerOnTime(command.getLong(1)); + break; + case REGISTER_NAMED: + timeContext.registerOnTime( + command.getString(2).toString(), command.getLong(1)); + break; + case DELETE_ANONYMOUS: + timeContext.clearTimer(command.getLong(1)); + break; + case DELETE_NAMED: + timeContext.clearTimer(command.getString(2).toString()); + break; + case CLEAR_ALL: + timeContext.clearAllTimers(); + break; + default: + throw new IllegalArgumentException( + "Unknown PTF timer operation: " + operation); + } + } + } catch (Exception e) { + throw new FlinkRuntimeException("Failed to apply a Python PTF timer command.", e); + } + } + + private RowData toBackendKey(RowData key) { + final TypeSerializer serializer = keyedStateBackend.getKeySerializer(); + if (serializer instanceof RowDataSerializer) { + return ((RowDataSerializer) serializer).toBinaryRow(key).copy(); + } + return key; + } + + private void setBackendCurrentKey(RowData key) { + setCurrentKeyForStreaming((KeyedStateBackend) keyedStateBackend, key); + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/PythonProcessTableFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/PythonProcessTableFunctionOperator.java new file mode 100644 index 0000000000000..f5d600a87ed5b --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/process/PythonProcessTableFunctionOperator.java @@ -0,0 +1,526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.state.MapState; +import org.apache.flink.api.common.state.MapStateDescriptor; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; +import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.core.memory.ManagedMemoryUseCase; +import org.apache.flink.fnexecution.v1.FlinkFnApi; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.runtime.state.KeyedStateBackend; +import org.apache.flink.runtime.state.VoidNamespace; +import org.apache.flink.runtime.state.VoidNamespaceSerializer; +import org.apache.flink.streaming.api.operators.InternalTimer; +import org.apache.flink.streaming.api.operators.InternalTimerService; +import org.apache.flink.streaming.api.operators.Triggerable; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.data.utils.JoinedRowData; +import org.apache.flink.table.functions.ProcessTableFunction; +import org.apache.flink.table.functions.python.PythonEnv; +import org.apache.flink.table.functions.python.PythonProcessTableFunction; +import org.apache.flink.table.runtime.generated.GeneratedProjection; +import org.apache.flink.table.runtime.generated.Projection; +import org.apache.flink.table.runtime.operators.process.RuntimeStateInfo; +import org.apache.flink.table.runtime.operators.process.RuntimeTableSemantics; +import org.apache.flink.table.runtime.operators.process.WritableInternalTimeContext; +import org.apache.flink.table.runtime.operators.python.AbstractOneInputPythonFunctionOperator; +import org.apache.flink.table.runtime.operators.python.utils.StreamRecordRowDataWrappingCollector; +import org.apache.flink.table.runtime.runners.python.beam.BeamProcessTableFunctionRunner; +import org.apache.flink.table.runtime.typeutils.PythonTypeUtils; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.runtime.typeutils.StringDataSerializer; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TinyIntType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.flink.types.RowKind; + +import javax.annotation.Nullable; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.List; +import java.util.Queue; +import java.util.stream.Collectors; + +import static org.apache.flink.python.PythonOptions.MAP_STATE_READ_CACHE_SIZE; +import static org.apache.flink.python.PythonOptions.MAP_STATE_WRITE_CACHE_SIZE; +import static org.apache.flink.python.PythonOptions.PYTHON_METRIC_ENABLED; +import static org.apache.flink.python.PythonOptions.PYTHON_PROFILE_ENABLED; +import static org.apache.flink.python.PythonOptions.STATE_CACHE_SIZE; +import static org.apache.flink.python.util.ProtoUtils.createFlattenRowTypeCoderInfoDescriptorProto; +import static org.apache.flink.python.util.ProtoUtils.createRowTypeCoderInfoDescriptorProto; +import static org.apache.flink.streaming.api.utils.PythonOperatorUtils.setCurrentKeyForStreaming; + +/** Runtime operator for a single-input Python {@link ProcessTableFunction}. */ +@Internal +public final class PythonProcessTableFunctionOperator + extends AbstractOneInputPythonFunctionOperator + implements Triggerable { + + private static final long serialVersionUID = 1L; + + private static final String NAMED_TIMER_STATE_NAME = "__flink_internal_python_ptf_named_timers"; + private static final String NAMED_TIMER_SERVICE_NAME = + "__flink_internal_python_ptf_named_timer_service"; + private static final String ANONYMOUS_TIMER_SERVICE_NAME = + "__flink_internal_python_ptf_anonymous_timer_service"; + + private final PythonProcessTableFunction function; + private final RuntimeTableSemantics tableSemantics; + private final List stateInfos; + private final RowType inputType; + private final RowType argumentType; + private final RowType resultType; + private final RowType keyType; + private final GeneratedProjection argumentGeneratedProjection; + + private transient Projection argumentProjection; + private transient TypeSerializer runnerInputSerializer; + private transient TypeSerializer resultSerializer; + private transient TypeSerializer timerDataSerializer; + private transient ByteArrayInputStreamWithPos bais; + private transient DataInputViewStreamWrapper baisWrapper; + private transient ByteArrayOutputStreamWithPos baos; + private transient DataOutputViewStreamWrapper baosWrapper; + private transient RowDataSerializer inputSerializer; + private transient RowDataSerializer keySerializer; + private transient TypeSerializer stateKeySerializer; + private transient StreamRecordRowDataWrappingCollector outputCollector; + private transient Queue invocations; + private transient Invocation currentInvocation; + private transient JoinedRowData withResult; + private transient JoinedRowData withRowtime; + private transient long inputWatermark; + private transient Object keyForTimerService; + + private transient @Nullable MapState namedTimers; + private transient @Nullable InternalTimerService namedTimerService; + private transient @Nullable InternalTimerService anonymousTimerService; + private transient @Nullable WritableInternalTimeContext timeContext; + private transient @Nullable ProcessTableTimerRegistration timerRegistration; + private transient RowType runnerInputType; + private transient RowType timerDataType; + + public PythonProcessTableFunctionOperator( + Configuration config, + PythonProcessTableFunction function, + RuntimeTableSemantics tableSemantics, + List stateInfos, + RowType inputType, + RowType argumentType, + RowType resultType, + RowType keyType, + GeneratedProjection argumentGeneratedProjection) { + super(config); + this.function = function; + this.tableSemantics = tableSemantics; + this.stateInfos = stateInfos; + this.inputType = inputType; + this.argumentType = argumentType; + this.resultType = resultType; + this.keyType = keyType; + this.argumentGeneratedProjection = argumentGeneratedProjection; + } + + @Override + public void open() throws Exception { + bais = new ByteArrayInputStreamWithPos(); + baisWrapper = new DataInputViewStreamWrapper(bais); + baos = new ByteArrayOutputStreamWithPos(); + baosWrapper = new DataOutputViewStreamWrapper(baos); + runnerInputType = createRunnerInputType(); + timerDataType = createTimerDataType(); + runnerInputSerializer = PythonTypeUtils.toInternalSerializer(runnerInputType); + resultSerializer = PythonTypeUtils.toInternalSerializer(resultType); + timerDataSerializer = PythonTypeUtils.toInternalSerializer(timerDataType); + inputSerializer = new RowDataSerializer(inputType); + keySerializer = + tableSemantics.hasSetSemantics() + ? (RowDataSerializer) + (TypeSerializer) getKeyedStateBackend().getKeySerializer() + : new RowDataSerializer(keyType); + stateKeySerializer = PythonTypeUtils.toInternalSerializer(keyType); + argumentProjection = + argumentGeneratedProjection.newInstance( + Thread.currentThread().getContextClassLoader()); + outputCollector = new StreamRecordRowDataWrappingCollector(output); + invocations = new ArrayDeque<>(); + withResult = new JoinedRowData(); + withRowtime = new JoinedRowData(); + inputWatermark = Long.MIN_VALUE; + setTimerServices(); + super.open(); + } + + @Override + public void processElement(StreamRecord element) throws Exception { + final RowData input = element.getValue(); + final RowData key = + tableSemantics.hasSetSemantics() ? (RowData) getCurrentKey() : GenericRowData.of(); + final RowData prefix = createPrefix(input, key); + final Long time = extractTime(input); + invocations.add(new Invocation(prefix, time)); + + final GenericRowData runnerInput = new GenericRowData(5); + runnerInput.setField(0, key); + runnerInput.setField(1, argumentProjection.apply(input)); + runnerInput.setField(2, time); + runnerInput.setField(3, nullableWatermark(inputWatermark)); + runnerInput.setField(4, nullableWatermark(currentWatermark())); + runnerInputSerializer.serialize(runnerInput, baosWrapper); + pythonFunctionRunner.process(baos.toByteArray()); + baos.reset(); + elementCount++; + checkInvokeFinishBundleByCount(); + emitResults(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + inputWatermark = mark.getTimestamp(); + super.processWatermark(mark); + } + + @Override + public void prepareSnapshotPreBarrier(long checkpointId) throws Exception { + super.prepareSnapshotPreBarrier(checkpointId); + drainUnregisteredTimers(); + } + + @Override + public void endInput() throws Exception { + super.endInput(); + drainUnregisteredTimers(); + } + + @Override + public void onEventTime(InternalTimer timer) throws Exception { + setBackendCurrentKey(timer.getKey()); + final boolean named = timer.getNamespace() != VoidNamespace.INSTANCE; + if (named) { + namedTimers.remove((StringData) timer.getNamespace()); + } + final RowData key = timer.getKey(); + invocations.add(new Invocation(keySerializer.copy(key), timer.getTimestamp())); + + final GenericRowData timerData = new GenericRowData(6); + timerData.setField(0, ProcessTableTimerRegistration.TRIGGER); + timerData.setField(1, timer.getTimestamp()); + timerData.setField(2, named ? timer.getNamespace() : null); + timerData.setField(3, key); + timerData.setField(4, null); + timerData.setField(5, nullableWatermark(currentWatermark())); + timerDataSerializer.serialize(timerData, baosWrapper); + pythonFunctionRunner.processTimer(baos.toByteArray()); + baos.reset(); + elementCount++; + checkInvokeFinishBundleByCount(); + emitResults(); + } + + @Override + public void onProcessingTime(InternalTimer timer) {} + + @Override + public PythonFunctionRunner createPythonFunctionRunner() throws Exception { + final boolean stateful = tableSemantics.hasSetSemantics(); + return new BeamProcessTableFunctionRunner( + getContainingTask().getEnvironment(), + getRuntimeContext().getTaskInfo().getTaskName(), + createPythonEnvironmentManager(), + createFunctionProto(), + getFlinkMetricContainer(), + stateful ? getKeyedStateBackend() : null, + stateful ? stateKeySerializer : null, + timerRegistration, + getContainingTask().getEnvironment().getMemoryManager(), + getOperatorConfig() + .getManagedMemoryFractionOperatorUseCaseOfSlot( + ManagedMemoryUseCase.PYTHON, + getContainingTask().getJobConfiguration(), + getContainingTask() + .getEnvironment() + .getTaskManagerInfo() + .getConfiguration(), + getContainingTask() + .getEnvironment() + .getUserCodeClassLoader() + .asClassLoader()), + createFlattenRowTypeCoderInfoDescriptorProto( + runnerInputType, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, true), + createFlattenRowTypeCoderInfoDescriptorProto( + resultType, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, true), + timerRegistration == null + ? null + : createRowTypeCoderInfoDescriptorProto( + timerDataType, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false)); + } + + @Override + public PythonEnv getPythonEnv() { + return function.getPythonEnv(); + } + + @Override + public void emitResult(Tuple3 resultTuple) throws Exception { + if (currentInvocation == null) { + currentInvocation = invocations.remove(); + } + while (resultTuple != null) { + if (isFinishResult(resultTuple)) { + currentInvocation = null; + return; + } + bais.setBuffer(resultTuple.f1, 0, resultTuple.f2); + final RowData result = resultSerializer.deserialize(baisWrapper); + if (result.getRowKind() != RowKind.INSERT) { + throw new TableRuntimeException( + "Python process table functions support append-only output."); + } + emitResultRow(currentInvocation, result); + resultTuple = pythonFunctionRunner.pollResult(); + } + } + + @Override + public void setCurrentKey(Object key) { + keyForTimerService = key; + } + + @Override + public Object getCurrentKey() { + return keyForTimerService; + } + + private void emitResultRow(Invocation invocation, RowData result) { + withResult.replace(invocation.prefix, result).setRowKind(RowKind.INSERT); + if (shouldEmitRowtime()) { + final GenericRowData rowtime = + GenericRowData.of( + invocation.time == null + ? null + : TimestampData.fromEpochMillis(invocation.time)); + withRowtime.replace(withResult, rowtime).setRowKind(RowKind.INSERT); + outputCollector.collect(withRowtime); + } else { + outputCollector.collect(withResult); + } + } + + private RowData createPrefix(RowData input, RowData key) { + if (tableSemantics.passColumnsThrough()) { + return inputSerializer.copy(input); + } + if (tableSemantics.hasSetSemantics()) { + return keySerializer.copy(key); + } + return GenericRowData.of(); + } + + private @Nullable Long extractTime(RowData input) { + final int timeColumn = tableSemantics.timeColumn(); + if (timeColumn < 0 || input.isNullAt(timeColumn)) { + return null; + } + final LogicalType timeType = inputType.getTypeAt(timeColumn); + final int precision; + if (timeType instanceof LocalZonedTimestampType) { + precision = ((LocalZonedTimestampType) timeType).getPrecision(); + } else { + precision = + ((org.apache.flink.table.types.logical.TimestampType) timeType).getPrecision(); + } + return input.getTimestamp(timeColumn, precision).getMillisecond(); + } + + private long currentWatermark() { + if (anonymousTimerService != null) { + return anonymousTimerService.currentWatermark(); + } + return inputWatermark; + } + + private static @Nullable Long nullableWatermark(long watermark) { + return watermark == Long.MIN_VALUE ? null : watermark; + } + + private boolean shouldEmitRowtime() { + return tableSemantics.timeColumn() >= 0; + } + + private static boolean isFinishResult(Tuple3 resultTuple) { + return resultTuple.f2 == 1 && resultTuple.f1[0] == 0x00; + } + + private void setBackendCurrentKey(RowData key) { + setCurrentKeyForStreaming( + (KeyedStateBackend) (KeyedStateBackend) getKeyedStateBackend(), key); + } + + private void setTimerServices() throws Exception { + if (!tableSemantics.hasSetSemantics() || !function.hasOnTimer()) { + return; + } + final MapStateDescriptor descriptor = + new MapStateDescriptor<>( + NAMED_TIMER_STATE_NAME, + StringDataSerializer.INSTANCE, + LongSerializer.INSTANCE); + namedTimers = getKeyedStateStore().getMapState(descriptor); + namedTimerService = + getInternalTimerService( + NAMED_TIMER_SERVICE_NAME, + StringDataSerializer.INSTANCE, + (Triggerable) this); + anonymousTimerService = + getInternalTimerService( + ANONYMOUS_TIMER_SERVICE_NAME, + VoidNamespaceSerializer.INSTANCE, + (Triggerable) this); + timeContext = + new WritableInternalTimeContext( + namedTimers, namedTimerService, anonymousTimerService); + timerRegistration = + new ProcessTableTimerRegistration( + this, + getKeyedStateBackend(), + timeContext, + timerDataSerializer, + keyType.getFieldCount()); + } + + private RowType createRunnerInputType() { + return RowType.of( + new LogicalType[] { + keyType, + argumentType, + new BigIntType(true), + new BigIntType(true), + new BigIntType(true) + }, + new String[] {"key", "arguments", "time", "table_watermark", "current_watermark"}); + } + + private RowType createTimerDataType() { + return RowType.of( + new LogicalType[] { + new TinyIntType(false), + new BigIntType(true), + new VarCharType(true, VarCharType.MAX_LENGTH), + keyType, + new BigIntType(true), + new BigIntType(true) + }, + new String[] { + "operation", "timestamp", "name", "key", "table_watermark", "current_watermark" + }); + } + + private FlinkFnApi.UserDefinedProcessTableFunction createFunctionProto() { + final FlinkFnApi.UserDefinedProcessTableFunction.Builder builder = + FlinkFnApi.UserDefinedProcessTableFunction.newBuilder() + .setPayload( + com.google.protobuf.ByteString.copyFrom( + function.getSerializedPythonFunction())) + .setHasOnTimer(function.hasOnTimer()) + .setMetricEnabled(config.get(PYTHON_METRIC_ENABLED)) + .setProfileEnabled(config.get(PYTHON_PROFILE_ENABLED)) + .setStateCacheSize(config.get(STATE_CACHE_SIZE)) + .setMapStateReadCacheSize(config.get(MAP_STATE_READ_CACHE_SIZE)) + .setMapStateWriteCacheSize(config.get(MAP_STATE_WRITE_CACHE_SIZE)); + if (tableSemantics.hasSetSemantics()) { + builder.setKeyType(PythonTypeUtils.toProtoType(keyType)); + } + final String[] names = function.getArgumentNames(); + final boolean[] tables = function.getTableArguments(); + final String[] traits = function.getArgumentTraits(); + for (int i = 0; i < names.length; i++) { + final FlinkFnApi.UserDefinedProcessTableFunction.Argument.Builder argument = + FlinkFnApi.UserDefinedProcessTableFunction.Argument.newBuilder() + .setName(names[i]) + .setType(PythonTypeUtils.toProtoType(argumentType.getTypeAt(i))) + .setIsTable(tables[i]); + if (!traits[i].isEmpty()) { + argument.addAllTraits(Arrays.asList(traits[i].split(","))); + } + builder.addArguments(argument); + } + for (RuntimeStateInfo stateInfo : stateInfos) { + builder.addStates( + FlinkFnApi.UserDefinedProcessTableFunction.State.newBuilder() + .setName(stateInfo.getStateName()) + .setType( + PythonTypeUtils.toProtoType( + stateInfo.getDataType().getLogicalType())) + .setTtlMillis(stateInfo.getTimeToLive())); + } + builder.addAllJobParameters( + getRuntimeContext().getGlobalJobParameters().entrySet().stream() + .map( + entry -> + FlinkFnApi.JobParameter.newBuilder() + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build()) + .collect(Collectors.toList())); + builder.setRuntimeContext( + FlinkFnApi.UserDefinedDataStreamFunction.RuntimeContext.newBuilder() + .setTaskName(getRuntimeContext().getTaskInfo().getTaskName()) + .setTaskNameWithSubtasks( + getRuntimeContext().getTaskInfo().getTaskNameWithSubtasks()) + .setNumberOfParallelSubtasks( + getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks()) + .setMaxNumberOfParallelSubtasks( + getRuntimeContext().getTaskInfo().getMaxNumberOfParallelSubtasks()) + .setIndexOfThisSubtask( + getRuntimeContext().getTaskInfo().getIndexOfThisSubtask()) + .setAttemptNumber(getRuntimeContext().getTaskInfo().getAttemptNumber()) + .build()); + return builder.build(); + } + + private static final class Invocation { + private final RowData prefix; + private final @Nullable Long time; + + private Invocation(RowData prefix, @Nullable Long time) { + this.prefix = prefix; + this.time = time; + } + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunner.java new file mode 100644 index 0000000000000..d1e9eb4f15c79 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunner.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.runners.python.beam; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.fnexecution.v1.FlinkFnApi; +import org.apache.flink.python.env.process.ProcessPythonEnvironmentManager; +import org.apache.flink.python.metric.process.FlinkMetricContainer; +import org.apache.flink.python.util.ProtoUtils; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.memory.MemoryManager; +import org.apache.flink.runtime.state.KeyedStateBackend; +import org.apache.flink.streaming.api.operators.python.process.timer.TimerRegistrationHandler; +import org.apache.flink.streaming.api.runners.python.beam.BeamPythonFunctionRunner; + +import com.google.protobuf.GeneratedMessage; +import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.apache.beam.runners.core.construction.BeamUrns; +import org.apache.beam.runners.core.construction.graph.TimerReference; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.apache.flink.python.Constants.INPUT_COLLECTION_ID; +import static org.apache.flink.python.Constants.MAIN_INPUT_NAME; +import static org.apache.flink.python.Constants.MAIN_OUTPUT_NAME; +import static org.apache.flink.python.Constants.OUTPUT_COLLECTION_ID; +import static org.apache.flink.python.Constants.TIMER_ID; +import static org.apache.flink.python.Constants.TRANSFORM_ID; +import static org.apache.flink.python.Constants.WRAPPER_TIMER_CODER_ID; + +/** Beam runner adapter for Python process table functions. */ +@Internal +public final class BeamProcessTableFunctionRunner extends BeamPythonFunctionRunner { + + public static final String PROCESS_TABLE_FUNCTION_URN = + "flink:transform:process_table_function:v1"; + + private final GeneratedMessage functionProto; + private final @Nullable FlinkFnApi.CoderInfoDescriptor timerCoderDescriptor; + + public BeamProcessTableFunctionRunner( + Environment environment, + String taskName, + ProcessPythonEnvironmentManager environmentManager, + GeneratedMessage functionProto, + @Nullable FlinkMetricContainer flinkMetricContainer, + @Nullable KeyedStateBackend keyedStateBackend, + @Nullable TypeSerializer keySerializer, + @Nullable TimerRegistrationHandler timerRegistration, + MemoryManager memoryManager, + double managedMemoryFraction, + FlinkFnApi.CoderInfoDescriptor inputCoderDescriptor, + FlinkFnApi.CoderInfoDescriptor outputCoderDescriptor, + @Nullable FlinkFnApi.CoderInfoDescriptor timerCoderDescriptor) { + super( + environment, + taskName, + environmentManager, + flinkMetricContainer, + keyedStateBackend, + null, + keySerializer, + null, + timerRegistration, + memoryManager, + managedMemoryFraction, + inputCoderDescriptor, + outputCoderDescriptor, + Collections.emptyMap()); + this.functionProto = functionProto; + this.timerCoderDescriptor = timerCoderDescriptor; + } + + @Override + protected void buildTransforms(RunnerApi.Components.Builder componentsBuilder) { + final RunnerApi.ParDoPayload.Builder payload = + RunnerApi.ParDoPayload.newBuilder() + .setDoFn( + RunnerApi.FunctionSpec.newBuilder() + .setUrn(PROCESS_TABLE_FUNCTION_URN) + .setPayload( + org.apache.beam.vendor.grpc.v1p60p1.com.google + .protobuf.ByteString.copyFrom( + functionProto.toByteArray())) + .build()); + if (timerCoderDescriptor != null) { + payload.putTimerFamilySpecs( + TIMER_ID, + RunnerApi.TimerFamilySpec.newBuilder() + .setTimeDomain(RunnerApi.TimeDomain.Enum.EVENT_TIME) + .setTimerFamilyCoderId(WRAPPER_TIMER_CODER_ID) + .build()); + } + + componentsBuilder.putTransforms( + TRANSFORM_ID, + RunnerApi.PTransform.newBuilder() + .setUniqueName(TRANSFORM_ID) + .setSpec( + RunnerApi.FunctionSpec.newBuilder() + .setUrn( + BeamUrns.getUrn( + RunnerApi.StandardPTransforms.Primitives + .PAR_DO)) + .setPayload(payload.build().toByteString()) + .build()) + .putInputs(MAIN_INPUT_NAME, INPUT_COLLECTION_ID) + .putOutputs(MAIN_OUTPUT_NAME, OUTPUT_COLLECTION_ID) + .build()); + } + + @Override + protected List getTimers(RunnerApi.Components components) { + if (timerCoderDescriptor == null) { + return Collections.emptyList(); + } + final RunnerApi.ExecutableStagePayload.TimerId timerId = + RunnerApi.ExecutableStagePayload.TimerId.newBuilder() + .setTransformId(TRANSFORM_ID) + .setLocalName(TIMER_ID) + .build(); + return Collections.singletonList(TimerReference.fromTimerId(timerId, components)); + } + + @Override + protected Optional getOptionalTimerCoderProto() { + if (timerCoderDescriptor == null) { + return Optional.empty(); + } + return Optional.of(ProtoUtils.createCoderProto(timerCoderDescriptor)); + } +} diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistrationTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistrationTest.java new file mode 100644 index 0000000000000..cb12ca53cfe84 --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/ProcessTableTimerRegistrationTest.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.api.common.state.MapState; +import org.apache.flink.api.common.state.MapStateDescriptor; +import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.runtime.state.VoidNamespace; +import org.apache.flink.runtime.state.VoidNamespaceSerializer; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.InternalTimer; +import org.apache.flink.streaming.api.operators.InternalTimerService; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.Triggerable; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.operators.process.WritableInternalTimeContext; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.runtime.typeutils.StringDataSerializer; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TinyIntType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.flink.util.FlinkRuntimeException; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.flink.streaming.api.utils.PythonOperatorUtils.setCurrentKeyForStreaming; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link ProcessTableTimerRegistration}. */ +class ProcessTableTimerRegistrationTest { + + private static final RowType KEY_TYPE = + RowType.of(new LogicalType[] {new VarCharType()}, new String[] {"key"}); + private static final RowType TIMER_DATA_TYPE = + RowType.of( + new LogicalType[] { + new TinyIntType(false), + new BigIntType(true), + new VarCharType(true, VarCharType.MAX_LENGTH), + KEY_TYPE, + new BigIntType(true), + new BigIntType(true) + }, + new String[] { + "operation", + "timestamp", + "name", + "key", + "table_watermark", + "current_watermark" + }); + + @Test + void testNamedAnonymousDeleteAndClearOperations() throws Exception { + final TestTimerOperator operator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(operator, null)) { + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 10L, "replace", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 20L, "replace", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 15L, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.DELETE_ANONYMOUS, 15L, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 18L, null, "A"); + + harness.processWatermark(new Watermark(19L)); + assertThat(operator.firedTimers).containsExactly("A::18"); + + harness.processWatermark(new Watermark(20L)); + assertThat(operator.firedTimers).containsExactly("A::18", "A:replace:20"); + } + } + + @Test + void testDeleteAndClearAllAreScopedToCurrentKey() throws Exception { + final TestTimerOperator operator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(operator, null)) { + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 10L, "delete", "A"); + operator.setTimer(ProcessTableTimerRegistration.DELETE_NAMED, null, "delete", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_NAMED, 11L, "clear", "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 12L, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.CLEAR_ALL, null, null, "A"); + operator.setTimer(ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 13L, null, "A"); + + harness.processWatermark(new Watermark(13L)); + + assertThat(operator.firedTimers).containsExactly("A::13"); + } + } + + @Test + void testTimersAreRestoredFromCheckpoint() throws Exception { + final OperatorSubtaskState snapshot; + final TestTimerOperator firstOperator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(firstOperator, null)) { + firstOperator.setTimer( + ProcessTableTimerRegistration.REGISTER_NAMED, 50L, "restored", "A"); + firstOperator.setTimer( + ProcessTableTimerRegistration.REGISTER_ANONYMOUS, 51L, null, "A"); + snapshot = harness.snapshot(1L, 1L); + } + + final TestTimerOperator restoredOperator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness(restoredOperator, snapshot)) { + harness.processWatermark(new Watermark(51L)); + + assertThat(restoredOperator.firedTimers) + .containsExactlyInAnyOrder("A:restored:50", "A::51"); + } + } + + @Test + void testRejectsUnknownTimerOperation() throws Exception { + final TestTimerOperator operator = new TestTimerOperator(); + try (KeyedOneInputStreamOperatorTestHarness ignored = + createHarness(operator, null)) { + assertThatThrownBy(() -> operator.setTimer((byte) 99, null, null, "A")) + .isInstanceOf(FlinkRuntimeException.class) + .hasMessageContaining("Failed to apply a Python PTF timer command") + .hasRootCauseMessage("Unknown PTF timer operation: 99"); + } + } + + private static KeyedOneInputStreamOperatorTestHarness createHarness( + TestTimerOperator operator, OperatorSubtaskState initialState) throws Exception { + final KeySelector keySelector = + value -> GenericRowData.of(value.getString(0)); + final KeyedOneInputStreamOperatorTestHarness harness = + new KeyedOneInputStreamOperatorTestHarness<>( + operator, keySelector, InternalTypeInfo.of(KEY_TYPE), 1, 1, 0); + harness.setup(new RowDataSerializer(KEY_TYPE)); + if (initialState != null) { + harness.initializeState(initialState); + } + harness.open(); + return harness; + } + + private static final class TestTimerOperator extends AbstractStreamOperator + implements OneInputStreamOperator, Triggerable { + + private static final long serialVersionUID = 1L; + + private final List firedTimers = new ArrayList<>(); + private transient MapState namedTimers; + private transient ProcessTableTimerRegistration registration; + private transient RowDataSerializer timerDataSerializer; + private transient Object keyForTimerService; + + @Override + public void open() throws Exception { + super.open(); + timerDataSerializer = new RowDataSerializer(TIMER_DATA_TYPE); + namedTimers = + getKeyedStateStore() + .getMapState( + new MapStateDescriptor<>( + "test-named-timers", + StringDataSerializer.INSTANCE, + LongSerializer.INSTANCE)); + final InternalTimerService namedTimerService = + getInternalTimerService( + "test-named-timer-service", + StringDataSerializer.INSTANCE, + (Triggerable) this); + final InternalTimerService anonymousTimerService = + getInternalTimerService( + "test-anonymous-timer-service", + VoidNamespaceSerializer.INSTANCE, + (Triggerable) this); + registration = + new ProcessTableTimerRegistration( + this, + getKeyedStateBackend(), + new WritableInternalTimeContext( + namedTimers, namedTimerService, anonymousTimerService), + timerDataSerializer, + 1); + } + + @Override + public void processElement(StreamRecord element) {} + + @Override + public void setCurrentKey(Object key) { + keyForTimerService = key; + } + + @Override + public Object getCurrentKey() { + return keyForTimerService; + } + + @Override + public void onEventTime(InternalTimer timer) throws Exception { + setCurrentKey(timer.getKey()); + setCurrentKeyForStreaming( + (org.apache.flink.runtime.state.KeyedStateBackend) + (org.apache.flink.runtime.state.KeyedStateBackend) + getKeyedStateBackend(), + timer.getKey()); + final String timerName; + if (timer.getNamespace() == VoidNamespace.INSTANCE) { + timerName = ""; + } else { + final StringData name = (StringData) timer.getNamespace(); + namedTimers.remove(name); + timerName = name.toString(); + } + firedTimers.add( + timer.getKey().getString(0) + ":" + timerName + ":" + timer.getTimestamp()); + } + + @Override + public void onProcessingTime(InternalTimer timer) {} + + private void setTimer(byte operation, Long timestamp, String name, String key) + throws Exception { + final GenericRowData command = new GenericRowData(6); + command.setField(0, operation); + command.setField(1, timestamp); + command.setField(2, name == null ? null : StringData.fromString(name)); + command.setField(3, GenericRowData.of(StringData.fromString(key))); + command.setField(4, null); + command.setField(5, null); + + final ByteArrayOutputStreamWithPos output = new ByteArrayOutputStreamWithPos(); + timerDataSerializer.serialize(command, new DataOutputViewStreamWrapper(output)); + registration.setTimer(output.toByteArray()); + } + } +} diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/TestJavaProcessTableFunctions.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/TestJavaProcessTableFunctions.java new file mode 100644 index 0000000000000..ef7d07324f641 --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/process/TestJavaProcessTableFunctions.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python.process; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.ArgumentTrait; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.functions.ProcessTableFunction; +import org.apache.flink.types.Row; + +/** Java process table functions used by the PyFlink API tests. */ +public final class TestJavaProcessTableFunctions { + + private TestJavaProcessTableFunctions() {} + + /** A PTF with one row-semantic table argument. */ + @DataTypeHint("ROW<`out` STRING>") + public static class RowSemanticFunction extends ProcessTableFunction { + public void eval( + @ArgumentHint(ArgumentTrait.ROW_SEMANTIC_TABLE) Row input, Integer increment) { + collect(Row.of(String.format("%s:%s", input.getFieldAs(0), increment))); + } + } + + /** A PTF with two set-semantic table arguments. */ + @DataTypeHint("ROW<`out` STRING>") + public static class MultiInputFunction extends ProcessTableFunction { + public void eval( + Context context, + @ArgumentHint(ArgumentTrait.SET_SEMANTIC_TABLE) Row in1, + @ArgumentHint({ + ArgumentTrait.SET_SEMANTIC_TABLE, + ArgumentTrait.OPTIONAL_PARTITION_BY + }) + Row in2) { + collect(Row.of(String.format("%s:%s", in1.getFieldAs(0), in2.getFieldAs(0)))); + } + } +} diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunnerTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunnerTest.java new file mode 100644 index 0000000000000..2cb5ae2ac25cb --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/runners/python/beam/BeamProcessTableFunctionRunnerTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.runners.python.beam; + +import org.apache.flink.fnexecution.v1.FlinkFnApi; +import org.apache.flink.python.Constants; +import org.apache.flink.table.runtime.utils.PythonTestUtils; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.RowType; + +import org.apache.beam.model.pipeline.v1.RunnerApi; +import org.junit.jupiter.api.Test; + +import static org.apache.flink.python.util.ProtoUtils.createFlattenRowTypeCoderInfoDescriptorProto; +import static org.apache.flink.python.util.ProtoUtils.createRowTypeCoderInfoDescriptorProto; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link BeamProcessTableFunctionRunner}. */ +class BeamProcessTableFunctionRunnerTest { + + private static final RowType DATA_TYPE = RowType.of(new BigIntType()); + + @Test + void testBuildsProcessTableFunctionTransformWithoutTimers() throws Exception { + final BeamProcessTableFunctionRunner runner = createRunner(false); + final RunnerApi.ParDoPayload payload = buildPayload(runner); + + assertThat(payload.getDoFn().getUrn()) + .isEqualTo(BeamProcessTableFunctionRunner.PROCESS_TABLE_FUNCTION_URN); + assertThat(payload.getTimerFamilySpecsMap()).isEmpty(); + assertThat(runner.getOptionalTimerCoderProto()).isEmpty(); + } + + @Test + void testDeclaresEventTimeTimerFamily() throws Exception { + final BeamProcessTableFunctionRunner runner = createRunner(true); + final RunnerApi.ParDoPayload payload = buildPayload(runner); + + assertThat(payload.getTimerFamilySpecsMap()).containsOnlyKeys(Constants.TIMER_ID); + assertThat(payload.getTimerFamilySpecsOrThrow(Constants.TIMER_ID).getTimeDomain()) + .isEqualTo(RunnerApi.TimeDomain.Enum.EVENT_TIME); + assertThat(runner.getOptionalTimerCoderProto()).isPresent(); + } + + private static RunnerApi.ParDoPayload buildPayload(BeamProcessTableFunctionRunner runner) + throws Exception { + final RunnerApi.Components.Builder components = RunnerApi.Components.newBuilder(); + runner.buildTransforms(components); + return RunnerApi.ParDoPayload.parseFrom( + components.getTransformsOrThrow(Constants.TRANSFORM_ID).getSpec().getPayload()); + } + + private static BeamProcessTableFunctionRunner createRunner(boolean withTimers) { + final FlinkFnApi.CoderInfoDescriptor dataCoder = + createFlattenRowTypeCoderInfoDescriptorProto( + DATA_TYPE, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, true); + return new BeamProcessTableFunctionRunner( + null, + "test-task", + PythonTestUtils.createTestProcessEnvironmentManager(), + FlinkFnApi.UserDefinedProcessTableFunction.getDefaultInstance(), + null, + null, + null, + null, + null, + 0.0, + dataCoder, + dataCoder, + withTimers + ? createRowTypeCoderInfoDescriptorProto( + DATA_TYPE, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false) + : null); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonProcessTableFunction.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonProcessTableFunction.java new file mode 100644 index 0000000000000..402939cf7c56b --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonProcessTableFunction.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.functions.python; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.functions.ProcessTableFunction; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.StateTypeStrategy; +import org.apache.flink.table.types.inference.StaticArgument; +import org.apache.flink.table.types.inference.StaticArgumentTrait; +import org.apache.flink.table.types.inference.TypeInference; +import org.apache.flink.table.types.inference.TypeStrategies; +import org.apache.flink.types.Row; +import org.apache.flink.util.Preconditions; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; + +/** Planner placeholder for a Python user-defined process table function. */ +@Internal +public class PythonProcessTableFunction extends ProcessTableFunction + implements PythonFunction { + + private static final long serialVersionUID = 1L; + + private final String name; + private final byte[] serializedProcessTableFunction; + private final String[] argumentNames; + private final DataType[] argumentDataTypes; + private final boolean[] tableArguments; + private final String[] argumentTraits; + private final String[] stateNames; + private final DataType[] stateDataTypes; + private final Duration[] stateTimeToLive; + private final DataType resultType; + private final boolean deterministic; + private final boolean hasOnTimer; + private final PythonEnv pythonEnv; + + public PythonProcessTableFunction( + String name, + byte[] serializedProcessTableFunction, + String[] argumentNames, + DataType[] argumentDataTypes, + boolean[] tableArguments, + String[] argumentTraits, + String[] stateNames, + DataType[] stateDataTypes, + Duration[] stateTimeToLive, + DataType resultType, + boolean deterministic, + boolean hasOnTimer, + PythonEnv pythonEnv) { + this.name = Preconditions.checkNotNull(name); + this.serializedProcessTableFunction = + Preconditions.checkNotNull(serializedProcessTableFunction); + this.argumentNames = Preconditions.checkNotNull(argumentNames); + this.argumentDataTypes = Preconditions.checkNotNull(argumentDataTypes); + this.tableArguments = Preconditions.checkNotNull(tableArguments); + this.argumentTraits = Preconditions.checkNotNull(argumentTraits); + this.stateNames = Preconditions.checkNotNull(stateNames); + this.stateDataTypes = Preconditions.checkNotNull(stateDataTypes); + this.stateTimeToLive = Preconditions.checkNotNull(stateTimeToLive); + this.resultType = Preconditions.checkNotNull(resultType); + this.deterministic = deterministic; + this.hasOnTimer = hasOnTimer; + this.pythonEnv = Preconditions.checkNotNull(pythonEnv); + validateMetadata(); + } + + public void eval(Object... args) { + throw new UnsupportedOperationException( + "This method is a placeholder and should not be called."); + } + + public void onTimer(Object... states) { + throw new UnsupportedOperationException( + "This method is a placeholder and should not be called."); + } + + @Override + public byte[] getSerializedPythonFunction() { + return serializedProcessTableFunction; + } + + @Override + public PythonEnv getPythonEnv() { + return pythonEnv; + } + + @Override + public boolean isDeterministic() { + return deterministic; + } + + public boolean hasOnTimer() { + return hasOnTimer; + } + + public String[] getArgumentNames() { + return argumentNames; + } + + public DataType[] getArgumentDataTypes() { + return argumentDataTypes; + } + + public boolean[] getTableArguments() { + return tableArguments; + } + + public String[] getArgumentTraits() { + return argumentTraits; + } + + public String[] getStateNames() { + return stateNames; + } + + public DataType[] getStateDataTypes() { + return stateDataTypes; + } + + public Duration[] getStateTimeToLive() { + return stateTimeToLive; + } + + public DataType getResultType() { + return resultType; + } + + @Override + public TypeInference getTypeInference(DataTypeFactory typeFactory) { + final List staticArguments = new ArrayList<>(); + for (int i = 0; i < argumentNames.length; i++) { + if (tableArguments[i]) { + final EnumSet traits = parseTraits(argumentTraits[i]); + final @Nullable DataType dataType = argumentDataTypes[i]; + if (dataType == null) { + staticArguments.add( + StaticArgument.table(argumentNames[i], Row.class, false, traits)); + } else { + staticArguments.add( + StaticArgument.table(argumentNames[i], dataType, false, traits)); + } + } else { + staticArguments.add( + StaticArgument.scalar(argumentNames[i], argumentDataTypes[i], false)); + } + } + + final LinkedHashMap stateStrategies = new LinkedHashMap<>(); + for (int i = 0; i < stateNames.length; i++) { + stateStrategies.put( + stateNames[i], + StateTypeStrategy.of( + TypeStrategies.explicit(stateDataTypes[i]), stateTimeToLive[i])); + } + + return TypeInference.newBuilder() + .staticArguments(staticArguments) + .stateTypeStrategies(stateStrategies) + .outputTypeStrategy(TypeStrategies.explicit(resultType)) + .build(); + } + + @Override + public String toString() { + return name; + } + + private void validateMetadata() { + Preconditions.checkArgument( + argumentNames.length == argumentDataTypes.length + && argumentNames.length == tableArguments.length + && argumentNames.length == argumentTraits.length, + "Argument metadata must have equal lengths."); + Preconditions.checkArgument( + stateNames.length == stateDataTypes.length + && stateNames.length == stateTimeToLive.length, + "State metadata must have equal lengths."); + } + + private static EnumSet parseTraits(String serializedTraits) { + final EnumSet traits = EnumSet.noneOf(StaticArgumentTrait.class); + if (serializedTraits.isEmpty()) { + return traits; + } + for (String trait : serializedTraits.split(",")) { + traits.add(StaticArgumentTrait.valueOf(trait)); + } + return traits; + } +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/functions/python/PythonProcessTableFunctionTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/functions/python/PythonProcessTableFunctionTest.java new file mode 100644 index 0000000000000..6145fa3bba47b --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/functions/python/PythonProcessTableFunctionTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.functions.python; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.inference.StaticArgument; +import org.apache.flink.table.types.inference.StaticArgumentTrait; +import org.apache.flink.table.types.inference.TypeInference; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link PythonProcessTableFunction}. */ +class PythonProcessTableFunctionTest { + + private static final DataType INPUT_TYPE = + DataTypes.ROW(DataTypes.FIELD("name", DataTypes.STRING())); + private static final DataType STATE_TYPE = + DataTypes.ROW(DataTypes.FIELD("count", DataTypes.BIGINT())); + private static final DataType RESULT_TYPE = + DataTypes.ROW(DataTypes.FIELD("result", DataTypes.STRING())); + + @Test + void testCreatesOrderedTypeInference() { + final PythonProcessTableFunction function = createFunction(); + + final TypeInference typeInference = function.getTypeInference(null); + final List arguments = typeInference.getStaticArguments().orElseThrow(); + + assertThat(arguments).extracting(StaticArgument::getName).containsExactly("input", "limit"); + assertThat(arguments.get(0).is(StaticArgumentTrait.TABLE)).isTrue(); + assertThat(arguments.get(0).is(StaticArgumentTrait.SET_SEMANTIC_TABLE)).isTrue(); + assertThat(arguments.get(0).is(StaticArgumentTrait.REQUIRE_ON_TIME)).isFalse(); + assertThat(arguments.get(1).is(StaticArgumentTrait.SCALAR)).isTrue(); + assertThat(typeInference.getStateTypeStrategies()).containsOnlyKeys("memory"); + assertThat(function.getStateTimeToLive()).containsExactly(Duration.ofDays(1)); + assertThat(function.hasOnTimer()).isTrue(); + assertThat(function.getPythonEnv().getExecType()).isEqualTo(PythonEnv.ExecType.PROCESS); + } + + @Test + void testSupportsPolymorphicTableArgument() { + final PythonProcessTableFunction function = + new PythonProcessTableFunction( + "polymorphic", + new byte[] {1}, + new String[] {"input"}, + new DataType[] {null}, + new boolean[] {true}, + new String[] {"ROW_SEMANTIC_TABLE"}, + new String[0], + new DataType[0], + new Duration[0], + RESULT_TYPE, + true, + false, + new PythonEnv(PythonEnv.ExecType.PROCESS)); + + final StaticArgument argument = + function.getTypeInference(null).getStaticArguments().orElseThrow().get(0); + + assertThat(argument.getName()).isEqualTo("input"); + assertThat(argument.is(StaticArgumentTrait.ROW_SEMANTIC_TABLE)).isTrue(); + } + + @Test + void testRejectsInconsistentArgumentMetadata() { + assertThatThrownBy( + () -> + new PythonProcessTableFunction( + "invalid", + new byte[] {1}, + new String[] {"input"}, + new DataType[0], + new boolean[] {true}, + new String[] {"SET_SEMANTIC_TABLE"}, + new String[0], + new DataType[0], + new Duration[0], + RESULT_TYPE, + true, + false, + new PythonEnv(PythonEnv.ExecType.PROCESS))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Argument metadata must have equal lengths"); + } + + private static PythonProcessTableFunction createFunction() { + return new PythonProcessTableFunction( + "test_ptf", + new byte[] {1, 2, 3}, + new String[] {"input", "limit"}, + new DataType[] {INPUT_TYPE, DataTypes.INT()}, + new boolean[] {true, false}, + new String[] {"SET_SEMANTIC_TABLE", ""}, + new String[] {"memory"}, + new DataType[] {STATE_TYPE}, + new Duration[] {Duration.ofDays(1)}, + RESULT_TYPE, + true, + true, + new PythonEnv(PythonEnv.ExecType.PROCESS)); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java index cbdadac3eb30a..bf455bc827f51 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/ReadableInternalTimeContext.java @@ -32,13 +32,13 @@ * {@link StaticArgumentTrait#PASS_COLUMNS_THROUGH}. */ @Internal -class ReadableInternalTimeContext implements ProcessTableFunction.TimeContext { +public class ReadableInternalTimeContext implements ProcessTableFunction.TimeContext { protected long tableWatermark; protected long currentWatermark; protected @Nullable Long time; - void setTime(long tableWatermark, long currentWatermark, @Nullable Long time) { + public void setTime(long tableWatermark, long currentWatermark, @Nullable Long time) { this.tableWatermark = tableWatermark; this.currentWatermark = currentWatermark; this.time = time; diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java index ce97c630c80c7..246edfdb87fcf 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/process/WritableInternalTimeContext.java @@ -35,13 +35,13 @@ /** Internal {@link TimeContext} for PTFs with set semantics. */ @Internal -class WritableInternalTimeContext extends ReadableInternalTimeContext { +public class WritableInternalTimeContext extends ReadableInternalTimeContext { private final MapState namedTimersMapState; private final InternalTimerService namedTimerService; private final InternalTimerService unnamedTimerService; - WritableInternalTimeContext( + public WritableInternalTimeContext( MapState namedTimersMapState, InternalTimerService namedTimerService, InternalTimerService unnamedTimerService) {