diff --git a/.claude/skills/blame/SKILL.md b/.claude/skills/blame/SKILL.md new file mode 100644 index 00000000000..83cdcfd5010 --- /dev/null +++ b/.claude/skills/blame/SKILL.md @@ -0,0 +1,12 @@ +--- +description: Show which re_gent step last modified each line of a file. Use when investigating file provenance or debugging. +allowed-tools: Bash(rgt blame *) +argument-hint: "[:]" +--- + +Display per-line provenance. + +Run: +```bash +rgt blame $ARGUMENTS +``` \ No newline at end of file diff --git a/.claude/skills/log/SKILL.md b/.claude/skills/log/SKILL.md new file mode 100644 index 00000000000..44e64b9560f --- /dev/null +++ b/.claude/skills/log/SKILL.md @@ -0,0 +1,23 @@ +--- +description: View the re_gent activity log for the default or selected session. The default view shows the conversation timeline and tool calls; file summaries are available with file flags. +allowed-tools: Bash(rgt log *) +argument-hint: "[session-id] [flags]" +--- + +Display the re_gent activity log. + +By default, `rgt log` shows the conversation timeline for the most recent session with captured steps. Use `--files-only` for file-change summaries. + +Run: +```bash +rgt log $ARGUMENTS +``` + +Common usage: +```bash +rgt log +rgt log --conversation-only +rgt log --files-only +rgt log --graph +rgt log --limit 50 +``` \ No newline at end of file diff --git a/.claude/skills/show/SKILL.md b/.claude/skills/show/SKILL.md new file mode 100644 index 00000000000..c11c55d7c85 --- /dev/null +++ b/.claude/skills/show/SKILL.md @@ -0,0 +1,12 @@ +--- +description: Show detailed context for a re_gent step, including tool calls, tool results, and conversation. +allowed-tools: Bash(rgt show *) +argument-hint: "" +--- + +Display full details for a step. + +Run: +```bash +rgt show $ARGUMENTS +``` \ No newline at end of file diff --git a/.gitignore b/.gitignore index f6e1955edc7..c8268bd4347 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,9 @@ # Ignoring AI agent files .agents/ + +# Compiled output of develop/streamdemo. +/streamdemo + +# Review scaffolding, kept out of the repo +/prompt.txt diff --git a/api/historyservice/v1/request_response.pb.go b/api/historyservice/v1/request_response.pb.go index a7f13f19b10..3b92b759aa9 100644 --- a/api/historyservice/v1/request_response.pb.go +++ b/api/historyservice/v1/request_response.pb.go @@ -11,29 +11,30 @@ import ( sync "sync" unsafe "unsafe" - v123 "go.temporal.io/api/activity/v1" + v124 "go.temporal.io/api/activity/v1" v14 "go.temporal.io/api/common/v1" v16 "go.temporal.io/api/deployment/v1" v12 "go.temporal.io/api/enums/v1" v13 "go.temporal.io/api/failure/v1" v17 "go.temporal.io/api/history/v1" - v121 "go.temporal.io/api/nexus/v1" + v122 "go.temporal.io/api/nexus/v1" v115 "go.temporal.io/api/protocol/v1" v114 "go.temporal.io/api/query/v1" + v116 "go.temporal.io/api/stream/v1" v111 "go.temporal.io/api/taskqueue/v1" v15 "go.temporal.io/api/workflow/v1" v1 "go.temporal.io/api/workflowservice/v1" - v118 "go.temporal.io/server/api/adminservice/v1" + v119 "go.temporal.io/server/api/adminservice/v1" v18 "go.temporal.io/server/api/clock/v1" - v119 "go.temporal.io/server/api/common/v1" + v120 "go.temporal.io/server/api/common/v1" v112 "go.temporal.io/server/api/enums/v1" - v122 "go.temporal.io/server/api/health/v1" + v123 "go.temporal.io/server/api/health/v1" v19 "go.temporal.io/server/api/history/v1" - v116 "go.temporal.io/server/api/namespace/v1" + v117 "go.temporal.io/server/api/namespace/v1" v110 "go.temporal.io/server/api/persistence/v1" - v117 "go.temporal.io/server/api/replication/v1" + v118 "go.temporal.io/server/api/replication/v1" v113 "go.temporal.io/server/api/taskqueue/v1" - v120 "go.temporal.io/server/api/token/v1" + v121 "go.temporal.io/server/api/token/v1" v11 "go.temporal.io/server/api/workflow/v1" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -1353,8 +1354,12 @@ type RecordWorkflowTaskStartedResponse struct { // Deprecated: Marked as deprecated in temporal/server/api/historyservice/v1/request_response.proto. RawHistory *v17.History `protobuf:"bytes,20,opt,name=raw_history,json=rawHistory,proto3" json:"raw_history,omitempty"` RawHistoryBytes [][]byte `protobuf:"bytes,21,rep,name=raw_history_bytes,json=rawHistoryBytes,proto3" json:"raw_history_bytes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + StreamSlices []*v116.StreamSlice `protobuf:"bytes,22,rep,name=stream_slices,json=streamSlices,proto3" json:"stream_slices,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RecordWorkflowTaskStartedResponse) Reset() { @@ -1528,6 +1533,13 @@ func (x *RecordWorkflowTaskStartedResponse) GetRawHistoryBytes() [][]byte { return nil } +func (x *RecordWorkflowTaskStartedResponse) GetStreamSlices() []*v116.StreamSlice { + if x != nil { + return x.StreamSlices + } + return nil +} + // RecordWorkflowTaskStartedResponseWithRawHistory is wire-compatible with RecordWorkflowTaskStartedResponse. // // WIRE COMPATIBILITY PATTERN: @@ -1572,8 +1584,12 @@ type RecordWorkflowTaskStartedResponseWithRawHistory struct { // Deprecated: Marked as deprecated in temporal/server/api/historyservice/v1/request_response.proto. RawHistory [][]byte `protobuf:"bytes,20,rep,name=raw_history,json=rawHistory,proto3" json:"raw_history,omitempty"` RawHistoryBytes [][]byte `protobuf:"bytes,21,rep,name=raw_history_bytes,json=rawHistoryBytes,proto3" json:"raw_history_bytes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + StreamSlices []*v116.StreamSlice `protobuf:"bytes,22,rep,name=stream_slices,json=streamSlices,proto3" json:"stream_slices,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RecordWorkflowTaskStartedResponseWithRawHistory) Reset() { @@ -1747,6 +1763,13 @@ func (x *RecordWorkflowTaskStartedResponseWithRawHistory) GetRawHistoryBytes() [ return nil } +func (x *RecordWorkflowTaskStartedResponseWithRawHistory) GetStreamSlices() []*v116.StreamSlice { + if x != nil { + return x.StreamSlices + } + return nil +} + type RecordActivityTaskStartedRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` @@ -5348,7 +5371,7 @@ type DescribeHistoryHostResponse struct { state protoimpl.MessageState `protogen:"open.v1"` ShardsNumber int32 `protobuf:"varint,1,opt,name=shards_number,json=shardsNumber,proto3" json:"shards_number,omitempty"` ShardIds []int32 `protobuf:"varint,2,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"` - NamespaceCache *v116.NamespaceCacheInfo `protobuf:"bytes,3,opt,name=namespace_cache,json=namespaceCache,proto3" json:"namespace_cache,omitempty"` + NamespaceCache *v117.NamespaceCacheInfo `protobuf:"bytes,3,opt,name=namespace_cache,json=namespaceCache,proto3" json:"namespace_cache,omitempty"` Address string `protobuf:"bytes,5,opt,name=address,proto3" json:"address,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -5398,7 +5421,7 @@ func (x *DescribeHistoryHostResponse) GetShardIds() []int32 { return nil } -func (x *DescribeHistoryHostResponse) GetNamespaceCache() *v116.NamespaceCacheInfo { +func (x *DescribeHistoryHostResponse) GetNamespaceCache() *v117.NamespaceCacheInfo { if x != nil { return x.NamespaceCache } @@ -5687,7 +5710,7 @@ func (*RemoveTaskResponse) Descriptor() ([]byte, []int) { type GetReplicationMessagesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Tokens []*v117.ReplicationToken `protobuf:"bytes,1,rep,name=tokens,proto3" json:"tokens,omitempty"` + Tokens []*v118.ReplicationToken `protobuf:"bytes,1,rep,name=tokens,proto3" json:"tokens,omitempty"` ClusterName string `protobuf:"bytes,2,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -5723,7 +5746,7 @@ func (*GetReplicationMessagesRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{76} } -func (x *GetReplicationMessagesRequest) GetTokens() []*v117.ReplicationToken { +func (x *GetReplicationMessagesRequest) GetTokens() []*v118.ReplicationToken { if x != nil { return x.Tokens } @@ -5739,7 +5762,7 @@ func (x *GetReplicationMessagesRequest) GetClusterName() string { type GetReplicationMessagesResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - ShardMessages map[int32]*v117.ReplicationMessages `protobuf:"bytes,1,rep,name=shard_messages,json=shardMessages,proto3" json:"shard_messages,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + ShardMessages map[int32]*v118.ReplicationMessages `protobuf:"bytes,1,rep,name=shard_messages,json=shardMessages,proto3" json:"shard_messages,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5774,7 +5797,7 @@ func (*GetReplicationMessagesResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{77} } -func (x *GetReplicationMessagesResponse) GetShardMessages() map[int32]*v117.ReplicationMessages { +func (x *GetReplicationMessagesResponse) GetShardMessages() map[int32]*v118.ReplicationMessages { if x != nil { return x.ShardMessages } @@ -5783,7 +5806,7 @@ func (x *GetReplicationMessagesResponse) GetShardMessages() map[int32]*v117.Repl type GetDLQReplicationMessagesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - TaskInfos []*v117.ReplicationTaskInfo `protobuf:"bytes,1,rep,name=task_infos,json=taskInfos,proto3" json:"task_infos,omitempty"` + TaskInfos []*v118.ReplicationTaskInfo `protobuf:"bytes,1,rep,name=task_infos,json=taskInfos,proto3" json:"task_infos,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5818,7 +5841,7 @@ func (*GetDLQReplicationMessagesRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{78} } -func (x *GetDLQReplicationMessagesRequest) GetTaskInfos() []*v117.ReplicationTaskInfo { +func (x *GetDLQReplicationMessagesRequest) GetTaskInfos() []*v118.ReplicationTaskInfo { if x != nil { return x.TaskInfos } @@ -5827,7 +5850,7 @@ func (x *GetDLQReplicationMessagesRequest) GetTaskInfos() []*v117.ReplicationTas type GetDLQReplicationMessagesResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - ReplicationTasks []*v117.ReplicationTask `protobuf:"bytes,1,rep,name=replication_tasks,json=replicationTasks,proto3" json:"replication_tasks,omitempty"` + ReplicationTasks []*v118.ReplicationTask `protobuf:"bytes,1,rep,name=replication_tasks,json=replicationTasks,proto3" json:"replication_tasks,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5862,7 +5885,7 @@ func (*GetDLQReplicationMessagesResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{79} } -func (x *GetDLQReplicationMessagesResponse) GetReplicationTasks() []*v117.ReplicationTask { +func (x *GetDLQReplicationMessagesResponse) GetReplicationTasks() []*v118.ReplicationTask { if x != nil { return x.ReplicationTasks } @@ -5968,7 +5991,7 @@ func (x *QueryWorkflowResponse) GetResponse() *v1.QueryWorkflowResponse { type ReapplyEventsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - Request *v118.ReapplyEventsRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + Request *v119.ReapplyEventsRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6010,7 +6033,7 @@ func (x *ReapplyEventsRequest) GetNamespaceId() string { return "" } -func (x *ReapplyEventsRequest) GetRequest() *v118.ReapplyEventsRequest { +func (x *ReapplyEventsRequest) GetRequest() *v119.ReapplyEventsRequest { if x != nil { return x.Request } @@ -6140,9 +6163,9 @@ func (x *GetDLQMessagesRequest) GetNextPageToken() []byte { type GetDLQMessagesResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Type v112.DeadLetterQueueType `protobuf:"varint,1,opt,name=type,proto3,enum=temporal.server.api.enums.v1.DeadLetterQueueType" json:"type,omitempty"` - ReplicationTasks []*v117.ReplicationTask `protobuf:"bytes,2,rep,name=replication_tasks,json=replicationTasks,proto3" json:"replication_tasks,omitempty"` + ReplicationTasks []*v118.ReplicationTask `protobuf:"bytes,2,rep,name=replication_tasks,json=replicationTasks,proto3" json:"replication_tasks,omitempty"` NextPageToken []byte `protobuf:"bytes,3,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` - ReplicationTasksInfo []*v117.ReplicationTaskInfo `protobuf:"bytes,4,rep,name=replication_tasks_info,json=replicationTasksInfo,proto3" json:"replication_tasks_info,omitempty"` + ReplicationTasksInfo []*v118.ReplicationTaskInfo `protobuf:"bytes,4,rep,name=replication_tasks_info,json=replicationTasksInfo,proto3" json:"replication_tasks_info,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6184,7 +6207,7 @@ func (x *GetDLQMessagesResponse) GetType() v112.DeadLetterQueueType { return v112.DeadLetterQueueType(0) } -func (x *GetDLQMessagesResponse) GetReplicationTasks() []*v117.ReplicationTask { +func (x *GetDLQMessagesResponse) GetReplicationTasks() []*v118.ReplicationTask { if x != nil { return x.ReplicationTasks } @@ -6198,7 +6221,7 @@ func (x *GetDLQMessagesResponse) GetNextPageToken() []byte { return nil } -func (x *GetDLQMessagesResponse) GetReplicationTasksInfo() []*v117.ReplicationTaskInfo { +func (x *GetDLQMessagesResponse) GetReplicationTasksInfo() []*v118.ReplicationTaskInfo { if x != nil { return x.ReplicationTasksInfo } @@ -6442,7 +6465,7 @@ type RefreshWorkflowTasksRequest struct { NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // (-- api-linter: core::0141::forbidden-types=disabled --) ArchetypeId uint32 `protobuf:"varint,3,opt,name=archetype_id,json=archetypeId,proto3" json:"archetype_id,omitempty"` - Request *v118.RefreshWorkflowTasksRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + Request *v119.RefreshWorkflowTasksRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6491,7 +6514,7 @@ func (x *RefreshWorkflowTasksRequest) GetArchetypeId() uint32 { return 0 } -func (x *RefreshWorkflowTasksRequest) GetRequest() *v118.RefreshWorkflowTasksRequest { +func (x *RefreshWorkflowTasksRequest) GetRequest() *v119.RefreshWorkflowTasksRequest { if x != nil { return x.Request } @@ -7395,7 +7418,7 @@ func (x *StreamWorkflowReplicationMessagesRequest) GetAttributes() isStreamWorkf return nil } -func (x *StreamWorkflowReplicationMessagesRequest) GetSyncReplicationState() *v117.SyncReplicationState { +func (x *StreamWorkflowReplicationMessagesRequest) GetSyncReplicationState() *v118.SyncReplicationState { if x != nil { if x, ok := x.Attributes.(*StreamWorkflowReplicationMessagesRequest_SyncReplicationState); ok { return x.SyncReplicationState @@ -7409,7 +7432,7 @@ type isStreamWorkflowReplicationMessagesRequest_Attributes interface { } type StreamWorkflowReplicationMessagesRequest_SyncReplicationState struct { - SyncReplicationState *v117.SyncReplicationState `protobuf:"bytes,1,opt,name=sync_replication_state,json=syncReplicationState,proto3,oneof"` + SyncReplicationState *v118.SyncReplicationState `protobuf:"bytes,1,opt,name=sync_replication_state,json=syncReplicationState,proto3,oneof"` } func (*StreamWorkflowReplicationMessagesRequest_SyncReplicationState) isStreamWorkflowReplicationMessagesRequest_Attributes() { @@ -7462,7 +7485,7 @@ func (x *StreamWorkflowReplicationMessagesResponse) GetAttributes() isStreamWork return nil } -func (x *StreamWorkflowReplicationMessagesResponse) GetMessages() *v117.WorkflowReplicationMessages { +func (x *StreamWorkflowReplicationMessagesResponse) GetMessages() *v118.WorkflowReplicationMessages { if x != nil { if x, ok := x.Attributes.(*StreamWorkflowReplicationMessagesResponse_Messages); ok { return x.Messages @@ -7476,7 +7499,7 @@ type isStreamWorkflowReplicationMessagesResponse_Attributes interface { } type StreamWorkflowReplicationMessagesResponse_Messages struct { - Messages *v117.WorkflowReplicationMessages `protobuf:"bytes,1,opt,name=messages,proto3,oneof"` + Messages *v118.WorkflowReplicationMessages `protobuf:"bytes,1,opt,name=messages,proto3,oneof"` } func (*StreamWorkflowReplicationMessagesResponse_Messages) isStreamWorkflowReplicationMessagesResponse_Attributes() { @@ -7837,7 +7860,7 @@ func (x *GetWorkflowExecutionHistoryReverseResponse) GetResponse() *v1.GetWorkfl type GetWorkflowExecutionRawHistoryV2Request struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - Request *v118.GetWorkflowExecutionRawHistoryV2Request `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + Request *v119.GetWorkflowExecutionRawHistoryV2Request `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7879,7 +7902,7 @@ func (x *GetWorkflowExecutionRawHistoryV2Request) GetNamespaceId() string { return "" } -func (x *GetWorkflowExecutionRawHistoryV2Request) GetRequest() *v118.GetWorkflowExecutionRawHistoryV2Request { +func (x *GetWorkflowExecutionRawHistoryV2Request) GetRequest() *v119.GetWorkflowExecutionRawHistoryV2Request { if x != nil { return x.Request } @@ -7888,7 +7911,7 @@ func (x *GetWorkflowExecutionRawHistoryV2Request) GetRequest() *v118.GetWorkflow type GetWorkflowExecutionRawHistoryV2Response struct { state protoimpl.MessageState `protogen:"open.v1"` - Response *v118.GetWorkflowExecutionRawHistoryV2Response `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` + Response *v119.GetWorkflowExecutionRawHistoryV2Response `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7923,7 +7946,7 @@ func (*GetWorkflowExecutionRawHistoryV2Response) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{117} } -func (x *GetWorkflowExecutionRawHistoryV2Response) GetResponse() *v118.GetWorkflowExecutionRawHistoryV2Response { +func (x *GetWorkflowExecutionRawHistoryV2Response) GetResponse() *v119.GetWorkflowExecutionRawHistoryV2Response { if x != nil { return x.Response } @@ -7933,7 +7956,7 @@ func (x *GetWorkflowExecutionRawHistoryV2Response) GetResponse() *v118.GetWorkfl type GetWorkflowExecutionRawHistoryRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - Request *v118.GetWorkflowExecutionRawHistoryRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + Request *v119.GetWorkflowExecutionRawHistoryRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -7975,7 +7998,7 @@ func (x *GetWorkflowExecutionRawHistoryRequest) GetNamespaceId() string { return "" } -func (x *GetWorkflowExecutionRawHistoryRequest) GetRequest() *v118.GetWorkflowExecutionRawHistoryRequest { +func (x *GetWorkflowExecutionRawHistoryRequest) GetRequest() *v119.GetWorkflowExecutionRawHistoryRequest { if x != nil { return x.Request } @@ -7984,7 +8007,7 @@ func (x *GetWorkflowExecutionRawHistoryRequest) GetRequest() *v118.GetWorkflowEx type GetWorkflowExecutionRawHistoryResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Response *v118.GetWorkflowExecutionRawHistoryResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` + Response *v119.GetWorkflowExecutionRawHistoryResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8019,7 +8042,7 @@ func (*GetWorkflowExecutionRawHistoryResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{119} } -func (x *GetWorkflowExecutionRawHistoryResponse) GetResponse() *v118.GetWorkflowExecutionRawHistoryResponse { +func (x *GetWorkflowExecutionRawHistoryResponse) GetResponse() *v119.GetWorkflowExecutionRawHistoryResponse { if x != nil { return x.Response } @@ -8031,7 +8054,7 @@ type ForceDeleteWorkflowExecutionRequest struct { NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // (-- api-linter: core::0141::forbidden-types=disabled --) ArchetypeId uint32 `protobuf:"varint,3,opt,name=archetype_id,json=archetypeId,proto3" json:"archetype_id,omitempty"` - Request *v118.DeleteWorkflowExecutionRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + Request *v119.DeleteWorkflowExecutionRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8080,7 +8103,7 @@ func (x *ForceDeleteWorkflowExecutionRequest) GetArchetypeId() uint32 { return 0 } -func (x *ForceDeleteWorkflowExecutionRequest) GetRequest() *v118.DeleteWorkflowExecutionRequest { +func (x *ForceDeleteWorkflowExecutionRequest) GetRequest() *v119.DeleteWorkflowExecutionRequest { if x != nil { return x.Request } @@ -8089,7 +8112,7 @@ func (x *ForceDeleteWorkflowExecutionRequest) GetRequest() *v118.DeleteWorkflowE type ForceDeleteWorkflowExecutionResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Response *v118.DeleteWorkflowExecutionResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` + Response *v119.DeleteWorkflowExecutionResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8124,7 +8147,7 @@ func (*ForceDeleteWorkflowExecutionResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{121} } -func (x *ForceDeleteWorkflowExecutionResponse) GetResponse() *v118.DeleteWorkflowExecutionResponse { +func (x *ForceDeleteWorkflowExecutionResponse) GetResponse() *v119.DeleteWorkflowExecutionResponse { if x != nil { return x.Response } @@ -8246,7 +8269,7 @@ func (*DeleteExecutionResponse) Descriptor() ([]byte, []int) { type GetDLQTasksRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - DlqKey *v119.HistoryDLQKey `protobuf:"bytes,1,opt,name=dlq_key,json=dlqKey,proto3" json:"dlq_key,omitempty"` + DlqKey *v120.HistoryDLQKey `protobuf:"bytes,1,opt,name=dlq_key,json=dlqKey,proto3" json:"dlq_key,omitempty"` // page_size must be positive. Up to this many tasks will be returned. PageSize int32 `protobuf:"varint,2,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` NextPageToken []byte `protobuf:"bytes,3,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` @@ -8284,7 +8307,7 @@ func (*GetDLQTasksRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{124} } -func (x *GetDLQTasksRequest) GetDlqKey() *v119.HistoryDLQKey { +func (x *GetDLQTasksRequest) GetDlqKey() *v120.HistoryDLQKey { if x != nil { return x.DlqKey } @@ -8307,7 +8330,7 @@ func (x *GetDLQTasksRequest) GetNextPageToken() []byte { type GetDLQTasksResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - DlqTasks []*v119.HistoryDLQTask `protobuf:"bytes,1,rep,name=dlq_tasks,json=dlqTasks,proto3" json:"dlq_tasks,omitempty"` + DlqTasks []*v120.HistoryDLQTask `protobuf:"bytes,1,rep,name=dlq_tasks,json=dlqTasks,proto3" json:"dlq_tasks,omitempty"` // next_page_token is empty if there are no more results. However, the converse is not true. If there are no more // results, this field may still be non-empty. This is to avoid having to do a count query to determine whether // there are more results. @@ -8346,7 +8369,7 @@ func (*GetDLQTasksResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{125} } -func (x *GetDLQTasksResponse) GetDlqTasks() []*v119.HistoryDLQTask { +func (x *GetDLQTasksResponse) GetDlqTasks() []*v120.HistoryDLQTask { if x != nil { return x.DlqTasks } @@ -8362,8 +8385,8 @@ func (x *GetDLQTasksResponse) GetNextPageToken() []byte { type DeleteDLQTasksRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - DlqKey *v119.HistoryDLQKey `protobuf:"bytes,1,opt,name=dlq_key,json=dlqKey,proto3" json:"dlq_key,omitempty"` - InclusiveMaxTaskMetadata *v119.HistoryDLQTaskMetadata `protobuf:"bytes,2,opt,name=inclusive_max_task_metadata,json=inclusiveMaxTaskMetadata,proto3" json:"inclusive_max_task_metadata,omitempty"` + DlqKey *v120.HistoryDLQKey `protobuf:"bytes,1,opt,name=dlq_key,json=dlqKey,proto3" json:"dlq_key,omitempty"` + InclusiveMaxTaskMetadata *v120.HistoryDLQTaskMetadata `protobuf:"bytes,2,opt,name=inclusive_max_task_metadata,json=inclusiveMaxTaskMetadata,proto3" json:"inclusive_max_task_metadata,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8398,14 +8421,14 @@ func (*DeleteDLQTasksRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{126} } -func (x *DeleteDLQTasksRequest) GetDlqKey() *v119.HistoryDLQKey { +func (x *DeleteDLQTasksRequest) GetDlqKey() *v120.HistoryDLQKey { if x != nil { return x.DlqKey } return nil } -func (x *DeleteDLQTasksRequest) GetInclusiveMaxTaskMetadata() *v119.HistoryDLQTaskMetadata { +func (x *DeleteDLQTasksRequest) GetInclusiveMaxTaskMetadata() *v120.HistoryDLQTaskMetadata { if x != nil { return x.InclusiveMaxTaskMetadata } @@ -8663,7 +8686,7 @@ func (*AddTasksResponse) Descriptor() ([]byte, []int) { type ListTasksRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Request *v118.ListHistoryTasksRequest `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + Request *v119.ListHistoryTasksRequest `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8698,7 +8721,7 @@ func (*ListTasksRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{132} } -func (x *ListTasksRequest) GetRequest() *v118.ListHistoryTasksRequest { +func (x *ListTasksRequest) GetRequest() *v119.ListHistoryTasksRequest { if x != nil { return x.Request } @@ -8707,7 +8730,7 @@ func (x *ListTasksRequest) GetRequest() *v118.ListHistoryTasksRequest { type ListTasksResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Response *v118.ListHistoryTasksResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` + Response *v119.ListHistoryTasksResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -8742,7 +8765,7 @@ func (*ListTasksResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{133} } -func (x *ListTasksResponse) GetResponse() *v118.ListHistoryTasksResponse { +func (x *ListTasksResponse) GetResponse() *v119.ListHistoryTasksResponse { if x != nil { return x.Response } @@ -8752,7 +8775,7 @@ func (x *ListTasksResponse) GetResponse() *v118.ListHistoryTasksResponse { type CompleteNexusOperationChasmRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Completion token - holds information for locating an entity and the corresponding component. - Completion *v120.NexusOperationCompletion `protobuf:"bytes,1,opt,name=completion,proto3" json:"completion,omitempty"` + Completion *v121.NexusOperationCompletion `protobuf:"bytes,1,opt,name=completion,proto3" json:"completion,omitempty"` // Types that are valid to be assigned to Outcome: // // *CompleteNexusOperationChasmRequest_Success @@ -8801,7 +8824,7 @@ func (*CompleteNexusOperationChasmRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{134} } -func (x *CompleteNexusOperationChasmRequest) GetCompletion() *v120.NexusOperationCompletion { +func (x *CompleteNexusOperationChasmRequest) GetCompletion() *v121.NexusOperationCompletion { if x != nil { return x.Completion } @@ -8918,7 +8941,7 @@ func (*CompleteNexusOperationChasmResponse) Descriptor() ([]byte, []int) { type CompleteNexusOperationRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Completion token - holds information for locating a run and the corresponding operation state machine. - Completion *v120.NexusOperationCompletion `protobuf:"bytes,1,opt,name=completion,proto3" json:"completion,omitempty"` + Completion *v121.NexusOperationCompletion `protobuf:"bytes,1,opt,name=completion,proto3" json:"completion,omitempty"` // Operation state - may only be successful / failed / canceled. State string `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"` // Types that are valid to be assigned to Outcome: @@ -8966,7 +8989,7 @@ func (*CompleteNexusOperationRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{136} } -func (x *CompleteNexusOperationRequest) GetCompletion() *v120.NexusOperationCompletion { +func (x *CompleteNexusOperationRequest) GetCompletion() *v121.NexusOperationCompletion { if x != nil { return x.Completion } @@ -8996,7 +9019,7 @@ func (x *CompleteNexusOperationRequest) GetSuccess() *v14.Payload { return nil } -func (x *CompleteNexusOperationRequest) GetFailure() *v121.Failure { +func (x *CompleteNexusOperationRequest) GetFailure() *v122.Failure { if x != nil { if x, ok := x.Outcome.(*CompleteNexusOperationRequest_Failure); ok { return x.Failure @@ -9037,7 +9060,7 @@ type CompleteNexusOperationRequest_Success struct { type CompleteNexusOperationRequest_Failure struct { // Operation failure, only set if state != successful. - Failure *v121.Failure `protobuf:"bytes,4,opt,name=failure,proto3,oneof"` + Failure *v122.Failure `protobuf:"bytes,4,opt,name=failure,proto3,oneof"` } func (*CompleteNexusOperationRequest_Success) isCompleteNexusOperationRequest_Outcome() {} @@ -9268,7 +9291,7 @@ type DeepHealthCheckResponse struct { state protoimpl.MessageState `protogen:"open.v1"` State v112.HealthState `protobuf:"varint,1,opt,name=state,proto3,enum=temporal.server.api.enums.v1.HealthState" json:"state,omitempty"` // Per-check diagnostic results. Populated for all checks regardless of state. - Checks []*v122.HealthCheck `protobuf:"bytes,2,rep,name=checks,proto3" json:"checks,omitempty"` + Checks []*v123.HealthCheck `protobuf:"bytes,2,rep,name=checks,proto3" json:"checks,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -9310,7 +9333,7 @@ func (x *DeepHealthCheckResponse) GetState() v112.HealthState { return v112.HealthState(0) } -func (x *DeepHealthCheckResponse) GetChecks() []*v122.HealthCheck { +func (x *DeepHealthCheckResponse) GetChecks() []*v123.HealthCheck { if x != nil { return x.Checks } @@ -9404,7 +9427,7 @@ func (x *SyncWorkflowStateRequest) GetArchetypeId() uint32 { type SyncWorkflowStateResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - VersionedTransitionArtifact *v117.VersionedTransitionArtifact `protobuf:"bytes,5,opt,name=versioned_transition_artifact,json=versionedTransitionArtifact,proto3" json:"versioned_transition_artifact,omitempty"` + VersionedTransitionArtifact *v118.VersionedTransitionArtifact `protobuf:"bytes,5,opt,name=versioned_transition_artifact,json=versionedTransitionArtifact,proto3" json:"versioned_transition_artifact,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -9439,7 +9462,7 @@ func (*SyncWorkflowStateResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{143} } -func (x *SyncWorkflowStateResponse) GetVersionedTransitionArtifact() *v117.VersionedTransitionArtifact { +func (x *SyncWorkflowStateResponse) GetVersionedTransitionArtifact() *v118.VersionedTransitionArtifact { if x != nil { return x.VersionedTransitionArtifact } @@ -9504,7 +9527,7 @@ func (x *UpdateActivityOptionsRequest) GetUpdateRequest() *v1.UpdateActivityOpti type UpdateActivityOptionsResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // Activity options after an update - ActivityOptions *v123.ActivityOptions `protobuf:"bytes,1,opt,name=activity_options,json=activityOptions,proto3" json:"activity_options,omitempty"` + ActivityOptions *v124.ActivityOptions `protobuf:"bytes,1,opt,name=activity_options,json=activityOptions,proto3" json:"activity_options,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -9539,7 +9562,7 @@ func (*UpdateActivityOptionsResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{145} } -func (x *UpdateActivityOptionsResponse) GetActivityOptions() *v123.ActivityOptions { +func (x *UpdateActivityOptionsResponse) GetActivityOptions() *v124.ActivityOptions { if x != nil { return x.ActivityOptions } @@ -10104,7 +10127,7 @@ type StartNexusOperationRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` ShardId int32 `protobuf:"varint,2,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` - Request *v121.StartOperationRequest `protobuf:"bytes,3,opt,name=request,proto3" json:"request,omitempty"` + Request *v122.StartOperationRequest `protobuf:"bytes,3,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -10153,7 +10176,7 @@ func (x *StartNexusOperationRequest) GetShardId() int32 { return 0 } -func (x *StartNexusOperationRequest) GetRequest() *v121.StartOperationRequest { +func (x *StartNexusOperationRequest) GetRequest() *v122.StartOperationRequest { if x != nil { return x.Request } @@ -10162,7 +10185,7 @@ func (x *StartNexusOperationRequest) GetRequest() *v121.StartOperationRequest { type StartNexusOperationResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Response *v121.StartOperationResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` + Response *v122.StartOperationResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -10197,7 +10220,7 @@ func (*StartNexusOperationResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{159} } -func (x *StartNexusOperationResponse) GetResponse() *v121.StartOperationResponse { +func (x *StartNexusOperationResponse) GetResponse() *v122.StartOperationResponse { if x != nil { return x.Response } @@ -10208,7 +10231,7 @@ type CancelNexusOperationRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` ShardId int32 `protobuf:"varint,2,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` - Request *v121.CancelOperationRequest `protobuf:"bytes,3,opt,name=request,proto3" json:"request,omitempty"` + Request *v122.CancelOperationRequest `protobuf:"bytes,3,opt,name=request,proto3" json:"request,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -10257,7 +10280,7 @@ func (x *CancelNexusOperationRequest) GetShardId() int32 { return 0 } -func (x *CancelNexusOperationRequest) GetRequest() *v121.CancelOperationRequest { +func (x *CancelNexusOperationRequest) GetRequest() *v122.CancelOperationRequest { if x != nil { return x.Request } @@ -10266,7 +10289,7 @@ func (x *CancelNexusOperationRequest) GetRequest() *v121.CancelOperationRequest type CancelNexusOperationResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Response *v121.CancelOperationResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` + Response *v122.CancelOperationResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -10301,7 +10324,7 @@ func (*CancelNexusOperationResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{161} } -func (x *CancelNexusOperationResponse) GetResponse() *v121.CancelOperationResponse { +func (x *CancelNexusOperationResponse) GetResponse() *v122.CancelOperationResponse { if x != nil { return x.Response } @@ -10709,7 +10732,7 @@ var File_temporal_server_api_historyservice_v1_request_response_proto protorefle const file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc = "" + "\n" + - " temporal.api.workflowservice.v1.StartWorkflowExecutionRequest @@ -11858,217 +11882,219 @@ var file_temporal_server_api_historyservice_v1_request_response_proto_depIdxs = 204, // 60: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.messages:type_name -> temporal.api.protocol.v1.Message 205, // 61: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.history:type_name -> temporal.api.history.v1.History 205, // 62: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.raw_history:type_name -> temporal.api.history.v1.History - 194, // 63: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 199, // 64: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo - 195, // 65: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 175, // 66: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 67: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.started_time:type_name -> google.protobuf.Timestamp - 167, // 68: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.queries:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry - 187, // 69: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 204, // 70: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.messages:type_name -> temporal.api.protocol.v1.Message - 205, // 71: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.history:type_name -> temporal.api.history.v1.History - 191, // 72: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 206, // 73: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueRequest - 187, // 74: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 201, // 75: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.build_id_redirect_info:type_name -> temporal.server.api.taskqueue.v1.BuildIdRedirectInfo - 202, // 76: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.scheduled_deployment:type_name -> temporal.api.deployment.v1.Deployment - 203, // 77: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective - 207, // 78: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.scheduled_event:type_name -> temporal.api.history.v1.HistoryEvent - 175, // 79: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.started_time:type_name -> google.protobuf.Timestamp - 175, // 80: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.current_attempt_scheduled_time:type_name -> google.protobuf.Timestamp - 178, // 81: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.heartbeat_details:type_name -> temporal.api.common.v1.Payloads - 194, // 82: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 187, // 83: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 208, // 84: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.priority:type_name -> temporal.api.common.v1.Priority - 209, // 85: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.retry_policy:type_name -> temporal.api.common.v1.RetryPolicy - 210, // 86: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest - 12, // 87: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.started_response:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse - 211, // 88: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.activity_tasks:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueResponse - 188, // 89: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.new_workflow_task:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse - 212, // 90: temporal.server.api.historyservice.v1.RespondWorkflowTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskFailedRequest - 191, // 91: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 92: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 213, // 93: temporal.server.api.historyservice.v1.RecordActivityTaskHeartbeatRequest.heartbeat_request:type_name -> temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatRequest - 214, // 94: temporal.server.api.historyservice.v1.RespondActivityTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCompletedRequest - 215, // 95: temporal.server.api.historyservice.v1.RespondActivityTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskFailedRequest - 216, // 96: temporal.server.api.historyservice.v1.RespondActivityTaskCanceledRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCanceledRequest - 191, // 97: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 98: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 217, // 99: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.signal_request:type_name -> temporal.api.workflowservice.v1.SignalWorkflowExecutionRequest - 191, // 100: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 190, // 101: temporal.server.api.historyservice.v1.SignalWorkflowExecutionResponse.link:type_name -> temporal.api.common.v1.Link - 218, // 102: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionRequest.signal_with_start_request:type_name -> temporal.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest - 190, // 103: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionResponse.signal_link:type_name -> temporal.api.common.v1.Link - 191, // 104: temporal.server.api.historyservice.v1.RemoveSignalMutableStateRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 219, // 105: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.terminate_request:type_name -> temporal.api.workflowservice.v1.TerminateWorkflowExecutionRequest - 191, // 106: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 107: temporal.server.api.historyservice.v1.DeleteWorkflowExecutionRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 220, // 108: temporal.server.api.historyservice.v1.ResetWorkflowExecutionRequest.reset_request:type_name -> temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest - 221, // 109: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RequestCancelWorkflowExecutionRequest - 191, // 110: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 111: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 112: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.child_clock:type_name -> temporal.server.api.clock.v1.VectorClock - 187, // 113: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.parent_clock:type_name -> temporal.server.api.clock.v1.VectorClock - 191, // 114: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 115: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 191, // 116: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 117: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 207, // 118: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.completion_event:type_name -> temporal.api.history.v1.HistoryEvent - 187, // 119: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 191, // 120: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 121: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 122: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 222, // 123: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest - 223, // 124: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.execution_config:type_name -> temporal.api.workflow.v1.WorkflowExecutionConfig - 224, // 125: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_execution_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionInfo - 225, // 126: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_activities:type_name -> temporal.api.workflow.v1.PendingActivityInfo - 226, // 127: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_children:type_name -> temporal.api.workflow.v1.PendingChildExecutionInfo - 227, // 128: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_workflow_task:type_name -> temporal.api.workflow.v1.PendingWorkflowTaskInfo - 228, // 129: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.callbacks:type_name -> temporal.api.workflow.v1.CallbackInfo - 229, // 130: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_nexus_operations:type_name -> temporal.api.workflow.v1.PendingNexusOperationInfo - 230, // 131: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_extended_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionExtendedInfo - 191, // 132: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 192, // 133: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.version_history_items:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 231, // 134: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.events:type_name -> temporal.api.common.v1.DataBlob - 231, // 135: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.new_run_events:type_name -> temporal.api.common.v1.DataBlob - 232, // 136: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo - 233, // 137: temporal.server.api.historyservice.v1.ReplicateWorkflowStateRequest.workflow_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 175, // 138: temporal.server.api.historyservice.v1.SyncShardStatusRequest.status_time:type_name -> google.protobuf.Timestamp - 175, // 139: temporal.server.api.historyservice.v1.SyncActivityRequest.scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 140: temporal.server.api.historyservice.v1.SyncActivityRequest.started_time:type_name -> google.protobuf.Timestamp - 175, // 141: temporal.server.api.historyservice.v1.SyncActivityRequest.last_heartbeat_time:type_name -> google.protobuf.Timestamp - 178, // 142: temporal.server.api.historyservice.v1.SyncActivityRequest.details:type_name -> temporal.api.common.v1.Payloads - 177, // 143: temporal.server.api.historyservice.v1.SyncActivityRequest.last_failure:type_name -> temporal.api.failure.v1.Failure - 234, // 144: temporal.server.api.historyservice.v1.SyncActivityRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 232, // 145: temporal.server.api.historyservice.v1.SyncActivityRequest.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo - 175, // 146: temporal.server.api.historyservice.v1.SyncActivityRequest.first_scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 147: temporal.server.api.historyservice.v1.SyncActivityRequest.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 179, // 148: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_initial_interval:type_name -> google.protobuf.Duration - 179, // 149: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_maximum_interval:type_name -> google.protobuf.Duration - 64, // 150: temporal.server.api.historyservice.v1.SyncActivitiesRequest.activities_info:type_name -> temporal.server.api.historyservice.v1.ActivitySyncInfo - 175, // 151: temporal.server.api.historyservice.v1.ActivitySyncInfo.scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 152: temporal.server.api.historyservice.v1.ActivitySyncInfo.started_time:type_name -> google.protobuf.Timestamp - 175, // 153: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_heartbeat_time:type_name -> google.protobuf.Timestamp - 178, // 154: temporal.server.api.historyservice.v1.ActivitySyncInfo.details:type_name -> temporal.api.common.v1.Payloads - 177, // 155: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_failure:type_name -> temporal.api.failure.v1.Failure - 234, // 156: temporal.server.api.historyservice.v1.ActivitySyncInfo.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 175, // 157: temporal.server.api.historyservice.v1.ActivitySyncInfo.first_scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 158: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 179, // 159: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_initial_interval:type_name -> google.protobuf.Duration - 179, // 160: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_maximum_interval:type_name -> google.protobuf.Duration - 191, // 161: temporal.server.api.historyservice.v1.DescribeMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 233, // 162: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.cache_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 233, // 163: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.database_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 191, // 164: temporal.server.api.historyservice.v1.DescribeHistoryHostRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 235, // 165: temporal.server.api.historyservice.v1.DescribeHistoryHostResponse.namespace_cache:type_name -> temporal.server.api.namespace.v1.NamespaceCacheInfo - 236, // 166: temporal.server.api.historyservice.v1.GetShardResponse.shard_info:type_name -> temporal.server.api.persistence.v1.ShardInfo - 175, // 167: temporal.server.api.historyservice.v1.RemoveTaskRequest.visibility_time:type_name -> google.protobuf.Timestamp - 237, // 168: temporal.server.api.historyservice.v1.GetReplicationMessagesRequest.tokens:type_name -> temporal.server.api.replication.v1.ReplicationToken - 168, // 169: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.shard_messages:type_name -> temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry - 238, // 170: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesRequest.task_infos:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo - 239, // 171: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask - 240, // 172: temporal.server.api.historyservice.v1.QueryWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.QueryWorkflowRequest - 241, // 173: temporal.server.api.historyservice.v1.QueryWorkflowResponse.response:type_name -> temporal.api.workflowservice.v1.QueryWorkflowResponse - 242, // 174: temporal.server.api.historyservice.v1.ReapplyEventsRequest.request:type_name -> temporal.server.api.adminservice.v1.ReapplyEventsRequest - 243, // 175: temporal.server.api.historyservice.v1.GetDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 243, // 176: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 239, // 177: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask - 238, // 178: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks_info:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo - 243, // 179: temporal.server.api.historyservice.v1.PurgeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 243, // 180: temporal.server.api.historyservice.v1.MergeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 244, // 181: temporal.server.api.historyservice.v1.RefreshWorkflowTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.RefreshWorkflowTasksRequest - 191, // 182: temporal.server.api.historyservice.v1.GenerateLastHistoryReplicationTasksRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 96, // 183: temporal.server.api.historyservice.v1.GetReplicationStatusResponse.shards:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus - 175, // 184: temporal.server.api.historyservice.v1.ShardReplicationStatus.shard_local_time:type_name -> google.protobuf.Timestamp - 169, // 185: temporal.server.api.historyservice.v1.ShardReplicationStatus.remote_clusters:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry - 170, // 186: temporal.server.api.historyservice.v1.ShardReplicationStatus.handover_namespaces:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry - 175, // 187: temporal.server.api.historyservice.v1.ShardReplicationStatus.max_replication_task_visibility_time:type_name -> google.protobuf.Timestamp - 175, // 188: temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster.acked_task_visibility_time:type_name -> google.protobuf.Timestamp - 191, // 189: temporal.server.api.historyservice.v1.RebuildMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 190: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 231, // 191: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.history_batches:type_name -> temporal.api.common.v1.DataBlob - 234, // 192: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 191, // 193: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 175, // 194: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_start_time:type_name -> google.protobuf.Timestamp - 175, // 195: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_close_time:type_name -> google.protobuf.Timestamp - 245, // 196: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionRequest - 246, // 197: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse.response:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionResponse - 247, // 198: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesRequest.sync_replication_state:type_name -> temporal.server.api.replication.v1.SyncReplicationState - 248, // 199: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesResponse.messages:type_name -> temporal.server.api.replication.v1.WorkflowReplicationMessages - 249, // 200: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateRequest - 250, // 201: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateResponse - 251, // 202: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest - 252, // 203: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse - 205, // 204: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.history:type_name -> temporal.api.history.v1.History - 252, // 205: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponseWithRaw.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse - 253, // 206: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseRequest - 254, // 207: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseResponse - 255, // 208: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Request.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Request - 256, // 209: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Response.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Response - 257, // 210: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryRequest.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryRequest - 258, // 211: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryResponse.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryResponse - 259, // 212: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionRequest.request:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionRequest - 260, // 213: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionResponse.response:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionResponse - 191, // 214: temporal.server.api.historyservice.v1.DeleteExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 261, // 215: temporal.server.api.historyservice.v1.GetDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey - 262, // 216: temporal.server.api.historyservice.v1.GetDLQTasksResponse.dlq_tasks:type_name -> temporal.server.api.common.v1.HistoryDLQTask - 261, // 217: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey - 263, // 218: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.inclusive_max_task_metadata:type_name -> temporal.server.api.common.v1.HistoryDLQTaskMetadata - 171, // 219: temporal.server.api.historyservice.v1.ListQueuesResponse.queues:type_name -> temporal.server.api.historyservice.v1.ListQueuesResponse.QueueInfo - 172, // 220: temporal.server.api.historyservice.v1.AddTasksRequest.tasks:type_name -> temporal.server.api.historyservice.v1.AddTasksRequest.Task - 264, // 221: temporal.server.api.historyservice.v1.ListTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksRequest - 265, // 222: temporal.server.api.historyservice.v1.ListTasksResponse.response:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksResponse - 266, // 223: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion - 267, // 224: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.success:type_name -> temporal.api.common.v1.Payload - 177, // 225: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.failure:type_name -> temporal.api.failure.v1.Failure - 175, // 226: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.close_time:type_name -> google.protobuf.Timestamp - 190, // 227: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.links:type_name -> temporal.api.common.v1.Link - 175, // 228: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.start_time:type_name -> google.protobuf.Timestamp - 266, // 229: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion - 267, // 230: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.success:type_name -> temporal.api.common.v1.Payload - 268, // 231: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.failure:type_name -> temporal.api.nexus.v1.Failure - 175, // 232: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.start_time:type_name -> google.protobuf.Timestamp - 190, // 233: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.links:type_name -> temporal.api.common.v1.Link - 269, // 234: temporal.server.api.historyservice.v1.InvokeStateMachineMethodRequest.ref:type_name -> temporal.server.api.persistence.v1.StateMachineRef - 270, // 235: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.state:type_name -> temporal.server.api.enums.v1.HealthState - 271, // 236: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.checks:type_name -> temporal.server.api.health.v1.HealthCheck - 191, // 237: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 193, // 238: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 197, // 239: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories - 272, // 240: temporal.server.api.historyservice.v1.SyncWorkflowStateResponse.versioned_transition_artifact:type_name -> temporal.server.api.replication.v1.VersionedTransitionArtifact - 273, // 241: temporal.server.api.historyservice.v1.UpdateActivityOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateActivityOptionsRequest - 274, // 242: temporal.server.api.historyservice.v1.UpdateActivityOptionsResponse.activity_options:type_name -> temporal.api.activity.v1.ActivityOptions - 275, // 243: temporal.server.api.historyservice.v1.PauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.PauseActivityRequest - 276, // 244: temporal.server.api.historyservice.v1.UnpauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.UnpauseActivityRequest - 277, // 245: temporal.server.api.historyservice.v1.ResetActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.ResetActivityRequest - 278, // 246: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionOptionsRequest - 279, // 247: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.workflow_execution_options:type_name -> temporal.api.workflow.v1.WorkflowExecutionOptions - 175, // 248: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.update_time:type_name -> google.protobuf.Timestamp - 280, // 249: temporal.server.api.historyservice.v1.PauseWorkflowExecutionRequest.pause_request:type_name -> temporal.api.workflowservice.v1.PauseWorkflowExecutionRequest - 281, // 250: temporal.server.api.historyservice.v1.UnpauseWorkflowExecutionRequest.unpause_request:type_name -> temporal.api.workflowservice.v1.UnpauseWorkflowExecutionRequest - 282, // 251: temporal.server.api.historyservice.v1.StartNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.StartOperationRequest - 283, // 252: temporal.server.api.historyservice.v1.StartNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.StartOperationResponse - 284, // 253: temporal.server.api.historyservice.v1.CancelNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.CancelOperationRequest - 285, // 254: temporal.server.api.historyservice.v1.CancelNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.CancelOperationResponse - 286, // 255: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingRequest - 287, // 256: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponse - 1, // 257: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest - 105, // 258: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest - 2, // 259: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse - 106, // 260: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse - 288, // 261: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery - 288, // 262: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery - 289, // 263: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry.value:type_name -> temporal.server.api.replication.v1.ReplicationMessages - 98, // 264: temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry.value:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster - 97, // 265: temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry.value:type_name -> temporal.server.api.historyservice.v1.HandoverNamespaceInfo - 231, // 266: temporal.server.api.historyservice.v1.AddTasksRequest.Task.blob:type_name -> temporal.api.common.v1.DataBlob - 290, // 267: temporal.server.api.historyservice.v1.routing:extendee -> google.protobuf.MessageOptions - 0, // 268: temporal.server.api.historyservice.v1.routing:type_name -> temporal.server.api.historyservice.v1.RoutingOptions - 269, // [269:269] is the sub-list for method output_type - 269, // [269:269] is the sub-list for method input_type - 268, // [268:269] is the sub-list for extension type_name - 267, // [267:268] is the sub-list for extension extendee - 0, // [0:267] is the sub-list for field type_name + 206, // 63: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.stream_slices:type_name -> temporal.api.stream.v1.StreamSlice + 194, // 64: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 199, // 65: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo + 195, // 66: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 175, // 67: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.scheduled_time:type_name -> google.protobuf.Timestamp + 175, // 68: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.started_time:type_name -> google.protobuf.Timestamp + 167, // 69: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.queries:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry + 187, // 70: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 204, // 71: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.messages:type_name -> temporal.api.protocol.v1.Message + 205, // 72: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.history:type_name -> temporal.api.history.v1.History + 206, // 73: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.stream_slices:type_name -> temporal.api.stream.v1.StreamSlice + 191, // 74: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 207, // 75: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueRequest + 187, // 76: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 201, // 77: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.build_id_redirect_info:type_name -> temporal.server.api.taskqueue.v1.BuildIdRedirectInfo + 202, // 78: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.scheduled_deployment:type_name -> temporal.api.deployment.v1.Deployment + 203, // 79: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective + 208, // 80: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.scheduled_event:type_name -> temporal.api.history.v1.HistoryEvent + 175, // 81: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.started_time:type_name -> google.protobuf.Timestamp + 175, // 82: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.current_attempt_scheduled_time:type_name -> google.protobuf.Timestamp + 178, // 83: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.heartbeat_details:type_name -> temporal.api.common.v1.Payloads + 194, // 84: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 187, // 85: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 209, // 86: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.priority:type_name -> temporal.api.common.v1.Priority + 210, // 87: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.retry_policy:type_name -> temporal.api.common.v1.RetryPolicy + 211, // 88: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest + 12, // 89: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.started_response:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse + 212, // 90: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.activity_tasks:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueResponse + 188, // 91: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.new_workflow_task:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse + 213, // 92: temporal.server.api.historyservice.v1.RespondWorkflowTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskFailedRequest + 191, // 93: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 187, // 94: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 214, // 95: temporal.server.api.historyservice.v1.RecordActivityTaskHeartbeatRequest.heartbeat_request:type_name -> temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatRequest + 215, // 96: temporal.server.api.historyservice.v1.RespondActivityTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCompletedRequest + 216, // 97: temporal.server.api.historyservice.v1.RespondActivityTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskFailedRequest + 217, // 98: temporal.server.api.historyservice.v1.RespondActivityTaskCanceledRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCanceledRequest + 191, // 99: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 187, // 100: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 218, // 101: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.signal_request:type_name -> temporal.api.workflowservice.v1.SignalWorkflowExecutionRequest + 191, // 102: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 190, // 103: temporal.server.api.historyservice.v1.SignalWorkflowExecutionResponse.link:type_name -> temporal.api.common.v1.Link + 219, // 104: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionRequest.signal_with_start_request:type_name -> temporal.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest + 190, // 105: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionResponse.signal_link:type_name -> temporal.api.common.v1.Link + 191, // 106: temporal.server.api.historyservice.v1.RemoveSignalMutableStateRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 220, // 107: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.terminate_request:type_name -> temporal.api.workflowservice.v1.TerminateWorkflowExecutionRequest + 191, // 108: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 191, // 109: temporal.server.api.historyservice.v1.DeleteWorkflowExecutionRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 221, // 110: temporal.server.api.historyservice.v1.ResetWorkflowExecutionRequest.reset_request:type_name -> temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest + 222, // 111: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RequestCancelWorkflowExecutionRequest + 191, // 112: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 191, // 113: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 187, // 114: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.child_clock:type_name -> temporal.server.api.clock.v1.VectorClock + 187, // 115: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.parent_clock:type_name -> temporal.server.api.clock.v1.VectorClock + 191, // 116: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 187, // 117: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 191, // 118: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 191, // 119: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 208, // 120: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.completion_event:type_name -> temporal.api.history.v1.HistoryEvent + 187, // 121: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 191, // 122: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 191, // 123: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 187, // 124: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 223, // 125: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest + 224, // 126: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.execution_config:type_name -> temporal.api.workflow.v1.WorkflowExecutionConfig + 225, // 127: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_execution_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionInfo + 226, // 128: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_activities:type_name -> temporal.api.workflow.v1.PendingActivityInfo + 227, // 129: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_children:type_name -> temporal.api.workflow.v1.PendingChildExecutionInfo + 228, // 130: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_workflow_task:type_name -> temporal.api.workflow.v1.PendingWorkflowTaskInfo + 229, // 131: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.callbacks:type_name -> temporal.api.workflow.v1.CallbackInfo + 230, // 132: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_nexus_operations:type_name -> temporal.api.workflow.v1.PendingNexusOperationInfo + 231, // 133: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_extended_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionExtendedInfo + 191, // 134: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 192, // 135: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.version_history_items:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 232, // 136: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.events:type_name -> temporal.api.common.v1.DataBlob + 232, // 137: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.new_run_events:type_name -> temporal.api.common.v1.DataBlob + 233, // 138: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo + 234, // 139: temporal.server.api.historyservice.v1.ReplicateWorkflowStateRequest.workflow_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 175, // 140: temporal.server.api.historyservice.v1.SyncShardStatusRequest.status_time:type_name -> google.protobuf.Timestamp + 175, // 141: temporal.server.api.historyservice.v1.SyncActivityRequest.scheduled_time:type_name -> google.protobuf.Timestamp + 175, // 142: temporal.server.api.historyservice.v1.SyncActivityRequest.started_time:type_name -> google.protobuf.Timestamp + 175, // 143: temporal.server.api.historyservice.v1.SyncActivityRequest.last_heartbeat_time:type_name -> google.protobuf.Timestamp + 178, // 144: temporal.server.api.historyservice.v1.SyncActivityRequest.details:type_name -> temporal.api.common.v1.Payloads + 177, // 145: temporal.server.api.historyservice.v1.SyncActivityRequest.last_failure:type_name -> temporal.api.failure.v1.Failure + 235, // 146: temporal.server.api.historyservice.v1.SyncActivityRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 233, // 147: temporal.server.api.historyservice.v1.SyncActivityRequest.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo + 175, // 148: temporal.server.api.historyservice.v1.SyncActivityRequest.first_scheduled_time:type_name -> google.protobuf.Timestamp + 175, // 149: temporal.server.api.historyservice.v1.SyncActivityRequest.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 179, // 150: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_initial_interval:type_name -> google.protobuf.Duration + 179, // 151: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_maximum_interval:type_name -> google.protobuf.Duration + 64, // 152: temporal.server.api.historyservice.v1.SyncActivitiesRequest.activities_info:type_name -> temporal.server.api.historyservice.v1.ActivitySyncInfo + 175, // 153: temporal.server.api.historyservice.v1.ActivitySyncInfo.scheduled_time:type_name -> google.protobuf.Timestamp + 175, // 154: temporal.server.api.historyservice.v1.ActivitySyncInfo.started_time:type_name -> google.protobuf.Timestamp + 175, // 155: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_heartbeat_time:type_name -> google.protobuf.Timestamp + 178, // 156: temporal.server.api.historyservice.v1.ActivitySyncInfo.details:type_name -> temporal.api.common.v1.Payloads + 177, // 157: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_failure:type_name -> temporal.api.failure.v1.Failure + 235, // 158: temporal.server.api.historyservice.v1.ActivitySyncInfo.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 175, // 159: temporal.server.api.historyservice.v1.ActivitySyncInfo.first_scheduled_time:type_name -> google.protobuf.Timestamp + 175, // 160: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 179, // 161: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_initial_interval:type_name -> google.protobuf.Duration + 179, // 162: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_maximum_interval:type_name -> google.protobuf.Duration + 191, // 163: temporal.server.api.historyservice.v1.DescribeMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 234, // 164: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.cache_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 234, // 165: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.database_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 191, // 166: temporal.server.api.historyservice.v1.DescribeHistoryHostRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 236, // 167: temporal.server.api.historyservice.v1.DescribeHistoryHostResponse.namespace_cache:type_name -> temporal.server.api.namespace.v1.NamespaceCacheInfo + 237, // 168: temporal.server.api.historyservice.v1.GetShardResponse.shard_info:type_name -> temporal.server.api.persistence.v1.ShardInfo + 175, // 169: temporal.server.api.historyservice.v1.RemoveTaskRequest.visibility_time:type_name -> google.protobuf.Timestamp + 238, // 170: temporal.server.api.historyservice.v1.GetReplicationMessagesRequest.tokens:type_name -> temporal.server.api.replication.v1.ReplicationToken + 168, // 171: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.shard_messages:type_name -> temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry + 239, // 172: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesRequest.task_infos:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo + 240, // 173: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask + 241, // 174: temporal.server.api.historyservice.v1.QueryWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.QueryWorkflowRequest + 242, // 175: temporal.server.api.historyservice.v1.QueryWorkflowResponse.response:type_name -> temporal.api.workflowservice.v1.QueryWorkflowResponse + 243, // 176: temporal.server.api.historyservice.v1.ReapplyEventsRequest.request:type_name -> temporal.server.api.adminservice.v1.ReapplyEventsRequest + 244, // 177: temporal.server.api.historyservice.v1.GetDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 244, // 178: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 240, // 179: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask + 239, // 180: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks_info:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo + 244, // 181: temporal.server.api.historyservice.v1.PurgeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 244, // 182: temporal.server.api.historyservice.v1.MergeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 245, // 183: temporal.server.api.historyservice.v1.RefreshWorkflowTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.RefreshWorkflowTasksRequest + 191, // 184: temporal.server.api.historyservice.v1.GenerateLastHistoryReplicationTasksRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 96, // 185: temporal.server.api.historyservice.v1.GetReplicationStatusResponse.shards:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus + 175, // 186: temporal.server.api.historyservice.v1.ShardReplicationStatus.shard_local_time:type_name -> google.protobuf.Timestamp + 169, // 187: temporal.server.api.historyservice.v1.ShardReplicationStatus.remote_clusters:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry + 170, // 188: temporal.server.api.historyservice.v1.ShardReplicationStatus.handover_namespaces:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry + 175, // 189: temporal.server.api.historyservice.v1.ShardReplicationStatus.max_replication_task_visibility_time:type_name -> google.protobuf.Timestamp + 175, // 190: temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster.acked_task_visibility_time:type_name -> google.protobuf.Timestamp + 191, // 191: temporal.server.api.historyservice.v1.RebuildMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 191, // 192: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 232, // 193: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.history_batches:type_name -> temporal.api.common.v1.DataBlob + 235, // 194: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 191, // 195: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 175, // 196: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_start_time:type_name -> google.protobuf.Timestamp + 175, // 197: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_close_time:type_name -> google.protobuf.Timestamp + 246, // 198: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionRequest + 247, // 199: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse.response:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionResponse + 248, // 200: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesRequest.sync_replication_state:type_name -> temporal.server.api.replication.v1.SyncReplicationState + 249, // 201: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesResponse.messages:type_name -> temporal.server.api.replication.v1.WorkflowReplicationMessages + 250, // 202: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateRequest + 251, // 203: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateResponse + 252, // 204: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest + 253, // 205: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse + 205, // 206: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.history:type_name -> temporal.api.history.v1.History + 253, // 207: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponseWithRaw.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse + 254, // 208: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseRequest + 255, // 209: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseResponse + 256, // 210: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Request.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Request + 257, // 211: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Response.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Response + 258, // 212: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryRequest.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryRequest + 259, // 213: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryResponse.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryResponse + 260, // 214: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionRequest.request:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionRequest + 261, // 215: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionResponse.response:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionResponse + 191, // 216: temporal.server.api.historyservice.v1.DeleteExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 262, // 217: temporal.server.api.historyservice.v1.GetDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey + 263, // 218: temporal.server.api.historyservice.v1.GetDLQTasksResponse.dlq_tasks:type_name -> temporal.server.api.common.v1.HistoryDLQTask + 262, // 219: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey + 264, // 220: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.inclusive_max_task_metadata:type_name -> temporal.server.api.common.v1.HistoryDLQTaskMetadata + 171, // 221: temporal.server.api.historyservice.v1.ListQueuesResponse.queues:type_name -> temporal.server.api.historyservice.v1.ListQueuesResponse.QueueInfo + 172, // 222: temporal.server.api.historyservice.v1.AddTasksRequest.tasks:type_name -> temporal.server.api.historyservice.v1.AddTasksRequest.Task + 265, // 223: temporal.server.api.historyservice.v1.ListTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksRequest + 266, // 224: temporal.server.api.historyservice.v1.ListTasksResponse.response:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksResponse + 267, // 225: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion + 268, // 226: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.success:type_name -> temporal.api.common.v1.Payload + 177, // 227: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.failure:type_name -> temporal.api.failure.v1.Failure + 175, // 228: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.close_time:type_name -> google.protobuf.Timestamp + 190, // 229: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.links:type_name -> temporal.api.common.v1.Link + 175, // 230: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.start_time:type_name -> google.protobuf.Timestamp + 267, // 231: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion + 268, // 232: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.success:type_name -> temporal.api.common.v1.Payload + 269, // 233: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.failure:type_name -> temporal.api.nexus.v1.Failure + 175, // 234: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.start_time:type_name -> google.protobuf.Timestamp + 190, // 235: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.links:type_name -> temporal.api.common.v1.Link + 270, // 236: temporal.server.api.historyservice.v1.InvokeStateMachineMethodRequest.ref:type_name -> temporal.server.api.persistence.v1.StateMachineRef + 271, // 237: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.state:type_name -> temporal.server.api.enums.v1.HealthState + 272, // 238: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.checks:type_name -> temporal.server.api.health.v1.HealthCheck + 191, // 239: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 193, // 240: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 197, // 241: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories + 273, // 242: temporal.server.api.historyservice.v1.SyncWorkflowStateResponse.versioned_transition_artifact:type_name -> temporal.server.api.replication.v1.VersionedTransitionArtifact + 274, // 243: temporal.server.api.historyservice.v1.UpdateActivityOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateActivityOptionsRequest + 275, // 244: temporal.server.api.historyservice.v1.UpdateActivityOptionsResponse.activity_options:type_name -> temporal.api.activity.v1.ActivityOptions + 276, // 245: temporal.server.api.historyservice.v1.PauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.PauseActivityRequest + 277, // 246: temporal.server.api.historyservice.v1.UnpauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.UnpauseActivityRequest + 278, // 247: temporal.server.api.historyservice.v1.ResetActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.ResetActivityRequest + 279, // 248: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionOptionsRequest + 280, // 249: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.workflow_execution_options:type_name -> temporal.api.workflow.v1.WorkflowExecutionOptions + 175, // 250: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.update_time:type_name -> google.protobuf.Timestamp + 281, // 251: temporal.server.api.historyservice.v1.PauseWorkflowExecutionRequest.pause_request:type_name -> temporal.api.workflowservice.v1.PauseWorkflowExecutionRequest + 282, // 252: temporal.server.api.historyservice.v1.UnpauseWorkflowExecutionRequest.unpause_request:type_name -> temporal.api.workflowservice.v1.UnpauseWorkflowExecutionRequest + 283, // 253: temporal.server.api.historyservice.v1.StartNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.StartOperationRequest + 284, // 254: temporal.server.api.historyservice.v1.StartNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.StartOperationResponse + 285, // 255: temporal.server.api.historyservice.v1.CancelNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.CancelOperationRequest + 286, // 256: temporal.server.api.historyservice.v1.CancelNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.CancelOperationResponse + 287, // 257: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingRequest + 288, // 258: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponse + 1, // 259: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest + 105, // 260: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest + 2, // 261: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse + 106, // 262: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse + 289, // 263: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery + 289, // 264: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery + 290, // 265: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry.value:type_name -> temporal.server.api.replication.v1.ReplicationMessages + 98, // 266: temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry.value:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster + 97, // 267: temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry.value:type_name -> temporal.server.api.historyservice.v1.HandoverNamespaceInfo + 232, // 268: temporal.server.api.historyservice.v1.AddTasksRequest.Task.blob:type_name -> temporal.api.common.v1.DataBlob + 291, // 269: temporal.server.api.historyservice.v1.routing:extendee -> google.protobuf.MessageOptions + 0, // 270: temporal.server.api.historyservice.v1.routing:type_name -> temporal.server.api.historyservice.v1.RoutingOptions + 271, // [271:271] is the sub-list for method output_type + 271, // [271:271] is the sub-list for method input_type + 270, // [270:271] is the sub-list for extension type_name + 269, // [269:270] is the sub-list for extension extendee + 0, // [0:269] is the sub-list for field type_name } func init() { file_temporal_server_api_historyservice_v1_request_response_proto_init() } diff --git a/api/matchingservice/v1/request_response.pb.go b/api/matchingservice/v1/request_response.pb.go index 4a2f7325773..bbdbe0254c9 100644 --- a/api/matchingservice/v1/request_response.pb.go +++ b/api/matchingservice/v1/request_response.pb.go @@ -12,22 +12,23 @@ import ( unsafe "unsafe" v11 "go.temporal.io/api/common/v1" - v112 "go.temporal.io/api/deployment/v1" - v19 "go.temporal.io/api/enums/v1" - v114 "go.temporal.io/api/failure/v1" + v113 "go.temporal.io/api/deployment/v1" + v110 "go.temporal.io/api/enums/v1" + v115 "go.temporal.io/api/failure/v1" v16 "go.temporal.io/api/history/v1" - v113 "go.temporal.io/api/nexus/v1" + v114 "go.temporal.io/api/nexus/v1" v15 "go.temporal.io/api/protocol/v1" v12 "go.temporal.io/api/query/v1" + v17 "go.temporal.io/api/stream/v1" v14 "go.temporal.io/api/taskqueue/v1" - v115 "go.temporal.io/api/worker/v1" + v116 "go.temporal.io/api/worker/v1" v1 "go.temporal.io/api/workflowservice/v1" - v17 "go.temporal.io/server/api/clock/v1" - v110 "go.temporal.io/server/api/deployment/v1" - v116 "go.temporal.io/server/api/enums/v1" + v18 "go.temporal.io/server/api/clock/v1" + v111 "go.temporal.io/server/api/deployment/v1" + v117 "go.temporal.io/server/api/enums/v1" v13 "go.temporal.io/server/api/history/v1" - v111 "go.temporal.io/server/api/persistence/v1" - v18 "go.temporal.io/server/api/taskqueue/v1" + v112 "go.temporal.io/server/api/persistence/v1" + v19 "go.temporal.io/server/api/taskqueue/v1" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" durationpb "google.golang.org/protobuf/types/known/durationpb" @@ -145,7 +146,11 @@ type PollWorkflowTaskQueueResponse struct { PollerScalingDecision *v14.PollerScalingDecision `protobuf:"bytes,21,opt,name=poller_scaling_decision,json=pollerScalingDecision,proto3" json:"poller_scaling_decision,omitempty"` // Raw history bytes sent from matching service when history.sendRawHistoryBetweenInternalServices is enabled. // Matching client will deserialize this to History when it receives the response. - RawHistory *v16.History `protobuf:"bytes,22,opt,name=raw_history,json=rawHistory,proto3" json:"raw_history,omitempty"` + RawHistory *v16.History `protobuf:"bytes,22,opt,name=raw_history,json=rawHistory,proto3" json:"raw_history,omitempty"` + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + StreamSlices []*v17.StreamSlice `protobuf:"bytes,23,rep,name=stream_slices,json=streamSlices,proto3" json:"stream_slices,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -327,6 +332,13 @@ func (x *PollWorkflowTaskQueueResponse) GetRawHistory() *v16.History { return nil } +func (x *PollWorkflowTaskQueueResponse) GetStreamSlices() []*v17.StreamSlice { + if x != nil { + return x.StreamSlices + } + return nil +} + // PollWorkflowTaskQueueResponseWithRawHistory is wire-compatible with PollWorkflowTaskQueueResponse. // // WIRE COMPATIBILITY PATTERN: @@ -374,7 +386,11 @@ type PollWorkflowTaskQueueResponseWithRawHistory struct { // Raw history bytes. Each element is a proto-encoded batch of history events. // When matching client deserializes this to PollWorkflowTaskQueueResponse, this field // will be automatically deserialized to the raw_history field as History. - RawHistory [][]byte `protobuf:"bytes,22,rep,name=raw_history,json=rawHistory,proto3" json:"raw_history,omitempty"` + RawHistory [][]byte `protobuf:"bytes,22,rep,name=raw_history,json=rawHistory,proto3" json:"raw_history,omitempty"` + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + StreamSlices []*v17.StreamSlice `protobuf:"bytes,23,rep,name=stream_slices,json=streamSlices,proto3" json:"stream_slices,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -556,6 +572,13 @@ func (x *PollWorkflowTaskQueueResponseWithRawHistory) GetRawHistory() [][]byte { return nil } +func (x *PollWorkflowTaskQueueResponseWithRawHistory) GetStreamSlices() []*v17.StreamSlice { + if x != nil { + return x.StreamSlices + } + return nil +} + type PollActivityTaskQueueRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` @@ -846,11 +869,11 @@ type AddWorkflowTaskRequest struct { // // aip.dev/not-precedent: "to" is used to indicate interval. --) ScheduleToStartTimeout *durationpb.Duration `protobuf:"bytes,5,opt,name=schedule_to_start_timeout,json=scheduleToStartTimeout,proto3" json:"schedule_to_start_timeout,omitempty"` - Clock *v17.VectorClock `protobuf:"bytes,9,opt,name=clock,proto3" json:"clock,omitempty"` + Clock *v18.VectorClock `protobuf:"bytes,9,opt,name=clock,proto3" json:"clock,omitempty"` // How this task should be directed by matching. (Missing means the default // for TaskVersionDirective, which is unversioned.) - VersionDirective *v18.TaskVersionDirective `protobuf:"bytes,10,opt,name=version_directive,json=versionDirective,proto3" json:"version_directive,omitempty"` - ForwardInfo *v18.TaskForwardInfo `protobuf:"bytes,11,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` + VersionDirective *v19.TaskVersionDirective `protobuf:"bytes,10,opt,name=version_directive,json=versionDirective,proto3" json:"version_directive,omitempty"` + ForwardInfo *v19.TaskForwardInfo `protobuf:"bytes,11,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` Priority *v11.Priority `protobuf:"bytes,12,opt,name=priority,proto3" json:"priority,omitempty"` // Stamp value from when the workflow task was scheduled. Used to validate the task is still relevant. Stamp int32 `protobuf:"varint,13,opt,name=stamp,proto3" json:"stamp,omitempty"` @@ -923,21 +946,21 @@ func (x *AddWorkflowTaskRequest) GetScheduleToStartTimeout() *durationpb.Duratio return nil } -func (x *AddWorkflowTaskRequest) GetClock() *v17.VectorClock { +func (x *AddWorkflowTaskRequest) GetClock() *v18.VectorClock { if x != nil { return x.Clock } return nil } -func (x *AddWorkflowTaskRequest) GetVersionDirective() *v18.TaskVersionDirective { +func (x *AddWorkflowTaskRequest) GetVersionDirective() *v19.TaskVersionDirective { if x != nil { return x.VersionDirective } return nil } -func (x *AddWorkflowTaskRequest) GetForwardInfo() *v18.TaskForwardInfo { +func (x *AddWorkflowTaskRequest) GetForwardInfo() *v19.TaskForwardInfo { if x != nil { return x.ForwardInfo } @@ -1014,11 +1037,11 @@ type AddActivityTaskRequest struct { // // aip.dev/not-precedent: "to" is used to indicate interval. --) ScheduleToStartTimeout *durationpb.Duration `protobuf:"bytes,6,opt,name=schedule_to_start_timeout,json=scheduleToStartTimeout,proto3" json:"schedule_to_start_timeout,omitempty"` - Clock *v17.VectorClock `protobuf:"bytes,9,opt,name=clock,proto3" json:"clock,omitempty"` + Clock *v18.VectorClock `protobuf:"bytes,9,opt,name=clock,proto3" json:"clock,omitempty"` // How this task should be directed by matching. (Missing means the default // for TaskVersionDirective, which is unversioned.) - VersionDirective *v18.TaskVersionDirective `protobuf:"bytes,10,opt,name=version_directive,json=versionDirective,proto3" json:"version_directive,omitempty"` - ForwardInfo *v18.TaskForwardInfo `protobuf:"bytes,11,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` + VersionDirective *v19.TaskVersionDirective `protobuf:"bytes,10,opt,name=version_directive,json=versionDirective,proto3" json:"version_directive,omitempty"` + ForwardInfo *v19.TaskForwardInfo `protobuf:"bytes,11,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` Stamp int32 `protobuf:"varint,12,opt,name=stamp,proto3" json:"stamp,omitempty"` Priority *v11.Priority `protobuf:"bytes,13,opt,name=priority,proto3" json:"priority,omitempty"` // Reference to the Chasm component for activity execution (if applicable). For standalone activities, all @@ -1094,21 +1117,21 @@ func (x *AddActivityTaskRequest) GetScheduleToStartTimeout() *durationpb.Duratio return nil } -func (x *AddActivityTaskRequest) GetClock() *v17.VectorClock { +func (x *AddActivityTaskRequest) GetClock() *v18.VectorClock { if x != nil { return x.Clock } return nil } -func (x *AddActivityTaskRequest) GetVersionDirective() *v18.TaskVersionDirective { +func (x *AddActivityTaskRequest) GetVersionDirective() *v19.TaskVersionDirective { if x != nil { return x.VersionDirective } return nil } -func (x *AddActivityTaskRequest) GetForwardInfo() *v18.TaskForwardInfo { +func (x *AddActivityTaskRequest) GetForwardInfo() *v19.TaskForwardInfo { if x != nil { return x.ForwardInfo } @@ -1189,8 +1212,8 @@ type QueryWorkflowRequest struct { QueryRequest *v1.QueryWorkflowRequest `protobuf:"bytes,3,opt,name=query_request,json=queryRequest,proto3" json:"query_request,omitempty"` // How this task should be directed by matching. (Missing means the default // for TaskVersionDirective, which is unversioned.) - VersionDirective *v18.TaskVersionDirective `protobuf:"bytes,5,opt,name=version_directive,json=versionDirective,proto3" json:"version_directive,omitempty"` - ForwardInfo *v18.TaskForwardInfo `protobuf:"bytes,6,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` + VersionDirective *v19.TaskVersionDirective `protobuf:"bytes,5,opt,name=version_directive,json=versionDirective,proto3" json:"version_directive,omitempty"` + ForwardInfo *v19.TaskForwardInfo `protobuf:"bytes,6,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` Priority *v11.Priority `protobuf:"bytes,7,opt,name=priority,proto3" json:"priority,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1247,14 +1270,14 @@ func (x *QueryWorkflowRequest) GetQueryRequest() *v1.QueryWorkflowRequest { return nil } -func (x *QueryWorkflowRequest) GetVersionDirective() *v18.TaskVersionDirective { +func (x *QueryWorkflowRequest) GetVersionDirective() *v19.TaskVersionDirective { if x != nil { return x.VersionDirective } return nil } -func (x *QueryWorkflowRequest) GetForwardInfo() *v18.TaskForwardInfo { +func (x *QueryWorkflowRequest) GetForwardInfo() *v19.TaskForwardInfo { if x != nil { return x.ForwardInfo } @@ -1427,7 +1450,7 @@ func (*RespondQueryTaskCompletedResponse) Descriptor() ([]byte, []int) { type CancelOutstandingPollRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - TaskQueueType v19.TaskQueueType `protobuf:"varint,2,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` + TaskQueueType v110.TaskQueueType `protobuf:"varint,2,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` TaskQueue *v14.TaskQueue `protobuf:"bytes,3,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` PollerId string `protobuf:"bytes,4,opt,name=poller_id,json=pollerId,proto3" json:"poller_id,omitempty"` unknownFields protoimpl.UnknownFields @@ -1471,11 +1494,11 @@ func (x *CancelOutstandingPollRequest) GetNamespaceId() string { return "" } -func (x *CancelOutstandingPollRequest) GetTaskQueueType() v19.TaskQueueType { +func (x *CancelOutstandingPollRequest) GetTaskQueueType() v110.TaskQueueType { if x != nil { return x.TaskQueueType } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } func (x *CancelOutstandingPollRequest) GetTaskQueue() *v14.TaskQueue { @@ -1533,7 +1556,7 @@ type CancelOutstandingWorkerPollsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` TaskQueue *v14.TaskQueue `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - TaskQueueType v19.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` + TaskQueueType v110.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` WorkerInstanceKey string `protobuf:"bytes,4,opt,name=worker_instance_key,json=workerInstanceKey,proto3" json:"worker_instance_key,omitempty"` // Worker identity string (e.g., "pid@hostname"). Used to eagerly remove the worker // from pollerHistory so DescribeTaskQueue doesn't show stale pollers. @@ -1588,11 +1611,11 @@ func (x *CancelOutstandingWorkerPollsRequest) GetTaskQueue() *v14.TaskQueue { return nil } -func (x *CancelOutstandingWorkerPollsRequest) GetTaskQueueType() v19.TaskQueueType { +func (x *CancelOutstandingWorkerPollsRequest) GetTaskQueueType() v110.TaskQueueType { if x != nil { return x.TaskQueueType } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } func (x *CancelOutstandingWorkerPollsRequest) GetWorkerInstanceKey() string { @@ -1661,9 +1684,9 @@ type CancelOutstandingWorkerPollsPartitionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // Used for RPC routing only. Set to any partition on the target host. - TaskQueuePartition *v18.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` + TaskQueuePartition *v19.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` // Partitions to process on this host. - Partitions []*v18.TaskQueuePartition `protobuf:"bytes,3,rep,name=partitions,proto3" json:"partitions,omitempty"` + Partitions []*v19.TaskQueuePartition `protobuf:"bytes,3,rep,name=partitions,proto3" json:"partitions,omitempty"` // Workers to cancel polls for. Workers []*CancelOutstandingWorkerPollsPartitionRequest_WorkerEntry `protobuf:"bytes,4,rep,name=workers,proto3" json:"workers,omitempty"` unknownFields protoimpl.UnknownFields @@ -1707,14 +1730,14 @@ func (x *CancelOutstandingWorkerPollsPartitionRequest) GetNamespaceId() string { return "" } -func (x *CancelOutstandingWorkerPollsPartitionRequest) GetTaskQueuePartition() *v18.TaskQueuePartition { +func (x *CancelOutstandingWorkerPollsPartitionRequest) GetTaskQueuePartition() *v19.TaskQueuePartition { if x != nil { return x.TaskQueuePartition } return nil } -func (x *CancelOutstandingWorkerPollsPartitionRequest) GetPartitions() []*v18.TaskQueuePartition { +func (x *CancelOutstandingWorkerPollsPartitionRequest) GetPartitions() []*v19.TaskQueuePartition { if x != nil { return x.Partitions } @@ -1776,7 +1799,7 @@ type DescribeTaskQueueRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` DescRequest *v1.DescribeTaskQueueRequest `protobuf:"bytes,2,opt,name=desc_request,json=descRequest,proto3" json:"desc_request,omitempty"` - Version *v110.WorkerDeploymentVersion `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + Version *v111.WorkerDeploymentVersion `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1825,7 +1848,7 @@ func (x *DescribeTaskQueueRequest) GetDescRequest() *v1.DescribeTaskQueueRequest return nil } -func (x *DescribeTaskQueueRequest) GetVersion() *v110.WorkerDeploymentVersion { +func (x *DescribeTaskQueueRequest) GetVersion() *v111.WorkerDeploymentVersion { if x != nil { return x.Version } @@ -1880,9 +1903,9 @@ type DescribeVersionedTaskQueuesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // This task queue is for routing purposes. - TaskQueueType v19.TaskQueueType `protobuf:"varint,2,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` + TaskQueueType v110.TaskQueueType `protobuf:"varint,2,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` TaskQueue *v14.TaskQueue `protobuf:"bytes,3,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - Version *v110.WorkerDeploymentVersion `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` + Version *v111.WorkerDeploymentVersion `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` // List of task queues to describe. VersionTaskQueues []*DescribeVersionedTaskQueuesRequest_VersionTaskQueue `protobuf:"bytes,5,rep,name=version_task_queues,json=versionTaskQueues,proto3" json:"version_task_queues,omitempty"` unknownFields protoimpl.UnknownFields @@ -1926,11 +1949,11 @@ func (x *DescribeVersionedTaskQueuesRequest) GetNamespaceId() string { return "" } -func (x *DescribeVersionedTaskQueuesRequest) GetTaskQueueType() v19.TaskQueueType { +func (x *DescribeVersionedTaskQueuesRequest) GetTaskQueueType() v110.TaskQueueType { if x != nil { return x.TaskQueueType } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } func (x *DescribeVersionedTaskQueuesRequest) GetTaskQueue() *v14.TaskQueue { @@ -1940,7 +1963,7 @@ func (x *DescribeVersionedTaskQueuesRequest) GetTaskQueue() *v14.TaskQueue { return nil } -func (x *DescribeVersionedTaskQueuesRequest) GetVersion() *v110.WorkerDeploymentVersion { +func (x *DescribeVersionedTaskQueuesRequest) GetVersion() *v111.WorkerDeploymentVersion { if x != nil { return x.Version } @@ -2001,7 +2024,7 @@ func (x *DescribeVersionedTaskQueuesResponse) GetVersionTaskQueues() []*Describe type DescribeTaskQueuePartitionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - TaskQueuePartition *v18.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` + TaskQueuePartition *v19.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` Versions *v14.TaskQueueVersionSelection `protobuf:"bytes,3,opt,name=versions,proto3" json:"versions,omitempty"` // Report task queue stats for the requested task queue types and versions ReportStats bool `protobuf:"varint,4,opt,name=report_stats,json=reportStats,proto3" json:"report_stats,omitempty"` @@ -2053,7 +2076,7 @@ func (x *DescribeTaskQueuePartitionRequest) GetNamespaceId() string { return "" } -func (x *DescribeTaskQueuePartitionRequest) GetTaskQueuePartition() *v18.TaskQueuePartition { +func (x *DescribeTaskQueuePartitionRequest) GetTaskQueuePartition() *v19.TaskQueuePartition { if x != nil { return x.TaskQueuePartition } @@ -2097,8 +2120,8 @@ func (x *DescribeTaskQueuePartitionRequest) GetOnlyIfLoaded() bool { type DescribeTaskQueuePartitionResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - VersionsInfoInternal map[string]*v18.TaskQueueVersionInfoInternal `protobuf:"bytes,1,rep,name=versions_info_internal,json=versionsInfoInternal,proto3" json:"versions_info_internal,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - ScaleInfo *v18.PartitionScaleInfo `protobuf:"bytes,2,opt,name=scale_info,json=scaleInfo,proto3" json:"scale_info,omitempty"` + VersionsInfoInternal map[string]*v19.TaskQueueVersionInfoInternal `protobuf:"bytes,1,rep,name=versions_info_internal,json=versionsInfoInternal,proto3" json:"versions_info_internal,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + ScaleInfo *v19.PartitionScaleInfo `protobuf:"bytes,2,opt,name=scale_info,json=scaleInfo,proto3" json:"scale_info,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2133,14 +2156,14 @@ func (*DescribeTaskQueuePartitionResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_matchingservice_v1_request_response_proto_rawDescGZIP(), []int{24} } -func (x *DescribeTaskQueuePartitionResponse) GetVersionsInfoInternal() map[string]*v18.TaskQueueVersionInfoInternal { +func (x *DescribeTaskQueuePartitionResponse) GetVersionsInfoInternal() map[string]*v19.TaskQueueVersionInfoInternal { if x != nil { return x.VersionsInfoInternal } return nil } -func (x *DescribeTaskQueuePartitionResponse) GetScaleInfo() *v18.PartitionScaleInfo { +func (x *DescribeTaskQueuePartitionResponse) GetScaleInfo() *v19.PartitionScaleInfo { if x != nil { return x.ScaleInfo } @@ -2779,8 +2802,8 @@ type GetTaskQueueUserDataRequest struct { NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // The task queue to fetch data from. The task queue is always considered as a normal // queue, since sticky queues have no user data. - TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - TaskQueueType v19.TaskQueueType `protobuf:"varint,5,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` + TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` + TaskQueueType v110.TaskQueueType `protobuf:"varint,5,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` // The value of the last known user data version. // If the requester has no data, it should set this to 0. // This value must not be set to a negative number (note that our linter suggests avoiding uint64). @@ -2843,11 +2866,11 @@ func (x *GetTaskQueueUserDataRequest) GetTaskQueue() string { return "" } -func (x *GetTaskQueueUserDataRequest) GetTaskQueueType() v19.TaskQueueType { +func (x *GetTaskQueueUserDataRequest) GetTaskQueueType() v110.TaskQueueType { if x != nil { return x.TaskQueueType } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } func (x *GetTaskQueueUserDataRequest) GetLastKnownUserDataVersion() int64 { @@ -2882,8 +2905,8 @@ type GetTaskQueueUserDataResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // Versioned user data, set if the task queue has user data and the request's last_known_user_data_version is less // than the version cached in the root partition. - UserData *v111.VersionedTaskQueueUserData `protobuf:"bytes,2,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` - EphemeralData *v18.VersionedEphemeralData `protobuf:"bytes,3,opt,name=ephemeral_data,json=ephemeralData,proto3" json:"ephemeral_data,omitempty"` + UserData *v112.VersionedTaskQueueUserData `protobuf:"bytes,2,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` + EphemeralData *v19.VersionedEphemeralData `protobuf:"bytes,3,opt,name=ephemeral_data,json=ephemeralData,proto3" json:"ephemeral_data,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2918,14 +2941,14 @@ func (*GetTaskQueueUserDataResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_matchingservice_v1_request_response_proto_rawDescGZIP(), []int{36} } -func (x *GetTaskQueueUserDataResponse) GetUserData() *v111.VersionedTaskQueueUserData { +func (x *GetTaskQueueUserDataResponse) GetUserData() *v112.VersionedTaskQueueUserData { if x != nil { return x.UserData } return nil } -func (x *GetTaskQueueUserDataResponse) GetEphemeralData() *v18.VersionedEphemeralData { +func (x *GetTaskQueueUserDataResponse) GetEphemeralData() *v19.VersionedEphemeralData { if x != nil { return x.EphemeralData } @@ -2940,8 +2963,8 @@ type SyncDeploymentUserDataRequest struct { // (-- api-linter: core::0203::required=disabled // // aip.dev/not-precedent: Not following Google API format --) - DeploymentName string `protobuf:"bytes,9,opt,name=deployment_name,json=deploymentName,proto3" json:"deployment_name,omitempty"` - TaskQueueTypes []v19.TaskQueueType `protobuf:"varint,8,rep,packed,name=task_queue_types,json=taskQueueTypes,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_types,omitempty"` + DeploymentName string `protobuf:"bytes,9,opt,name=deployment_name,json=deploymentName,proto3" json:"deployment_name,omitempty"` + TaskQueueTypes []v110.TaskQueueType `protobuf:"varint,8,rep,packed,name=task_queue_types,json=taskQueueTypes,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_types,omitempty"` // Types that are valid to be assigned to Operation: // // *SyncDeploymentUserDataRequest_UpdateVersionData @@ -2949,12 +2972,12 @@ type SyncDeploymentUserDataRequest struct { Operation isSyncDeploymentUserDataRequest_Operation `protobuf_oneof:"operation"` // Absent means no change. // Ignored by the task queue if new revision number is not greater that what it has. - UpdateRoutingConfig *v112.RoutingConfig `protobuf:"bytes,10,opt,name=update_routing_config,json=updateRoutingConfig,proto3" json:"update_routing_config,omitempty"` + UpdateRoutingConfig *v113.RoutingConfig `protobuf:"bytes,10,opt,name=update_routing_config,json=updateRoutingConfig,proto3" json:"update_routing_config,omitempty"` // Optional map of build id to upsert version data. // (-- api-linter: core::0203::required=disabled // // aip.dev/not-precedent: Not following Google API format --) - UpsertVersionsData map[string]*v110.WorkerDeploymentVersionData `protobuf:"bytes,11,rep,name=upsert_versions_data,json=upsertVersionsData,proto3" json:"upsert_versions_data,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + UpsertVersionsData map[string]*v111.WorkerDeploymentVersionData `protobuf:"bytes,11,rep,name=upsert_versions_data,json=upsertVersionsData,proto3" json:"upsert_versions_data,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // List of build ids to forget from task queue. ForgetVersions []string `protobuf:"bytes,12,rep,name=forget_versions,json=forgetVersions,proto3" json:"forget_versions,omitempty"` unknownFields protoimpl.UnknownFields @@ -3012,7 +3035,7 @@ func (x *SyncDeploymentUserDataRequest) GetDeploymentName() string { return "" } -func (x *SyncDeploymentUserDataRequest) GetTaskQueueTypes() []v19.TaskQueueType { +func (x *SyncDeploymentUserDataRequest) GetTaskQueueTypes() []v110.TaskQueueType { if x != nil { return x.TaskQueueTypes } @@ -3027,7 +3050,7 @@ func (x *SyncDeploymentUserDataRequest) GetOperation() isSyncDeploymentUserDataR } // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. -func (x *SyncDeploymentUserDataRequest) GetUpdateVersionData() *v110.DeploymentVersionData { +func (x *SyncDeploymentUserDataRequest) GetUpdateVersionData() *v111.DeploymentVersionData { if x != nil { if x, ok := x.Operation.(*SyncDeploymentUserDataRequest_UpdateVersionData); ok { return x.UpdateVersionData @@ -3037,7 +3060,7 @@ func (x *SyncDeploymentUserDataRequest) GetUpdateVersionData() *v110.DeploymentV } // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. -func (x *SyncDeploymentUserDataRequest) GetForgetVersion() *v110.WorkerDeploymentVersion { +func (x *SyncDeploymentUserDataRequest) GetForgetVersion() *v111.WorkerDeploymentVersion { if x != nil { if x, ok := x.Operation.(*SyncDeploymentUserDataRequest_ForgetVersion); ok { return x.ForgetVersion @@ -3046,14 +3069,14 @@ func (x *SyncDeploymentUserDataRequest) GetForgetVersion() *v110.WorkerDeploymen return nil } -func (x *SyncDeploymentUserDataRequest) GetUpdateRoutingConfig() *v112.RoutingConfig { +func (x *SyncDeploymentUserDataRequest) GetUpdateRoutingConfig() *v113.RoutingConfig { if x != nil { return x.UpdateRoutingConfig } return nil } -func (x *SyncDeploymentUserDataRequest) GetUpsertVersionsData() map[string]*v110.WorkerDeploymentVersionData { +func (x *SyncDeploymentUserDataRequest) GetUpsertVersionsData() map[string]*v111.WorkerDeploymentVersionData { if x != nil { return x.UpsertVersionsData } @@ -3075,14 +3098,14 @@ type SyncDeploymentUserDataRequest_UpdateVersionData struct { // The deployment version and its data that is being updated. // // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. - UpdateVersionData *v110.DeploymentVersionData `protobuf:"bytes,6,opt,name=update_version_data,json=updateVersionData,proto3,oneof"` + UpdateVersionData *v111.DeploymentVersionData `protobuf:"bytes,6,opt,name=update_version_data,json=updateVersionData,proto3,oneof"` } type SyncDeploymentUserDataRequest_ForgetVersion struct { // The version whose data should be cleaned from the task queue. // // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. - ForgetVersion *v110.WorkerDeploymentVersion `protobuf:"bytes,7,opt,name=forget_version,json=forgetVersion,proto3,oneof"` + ForgetVersion *v111.WorkerDeploymentVersion `protobuf:"bytes,7,opt,name=forget_version,json=forgetVersion,proto3,oneof"` } func (*SyncDeploymentUserDataRequest_UpdateVersionData) isSyncDeploymentUserDataRequest_Operation() {} @@ -3151,7 +3174,7 @@ type ApplyTaskQueueUserDataReplicationEventRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - UserData *v111.TaskQueueUserData `protobuf:"bytes,3,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` + UserData *v112.TaskQueueUserData `protobuf:"bytes,3,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3200,7 +3223,7 @@ func (x *ApplyTaskQueueUserDataReplicationEventRequest) GetTaskQueue() string { return "" } -func (x *ApplyTaskQueueUserDataReplicationEventRequest) GetUserData() *v111.TaskQueueUserData { +func (x *ApplyTaskQueueUserDataReplicationEventRequest) GetUserData() *v112.TaskQueueUserData { if x != nil { return x.UserData } @@ -3342,7 +3365,7 @@ func (x *GetBuildIdTaskQueueMappingResponse) GetTaskQueues() []string { type ForceLoadTaskQueuePartitionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - TaskQueuePartition *v18.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` + TaskQueuePartition *v19.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3384,7 +3407,7 @@ func (x *ForceLoadTaskQueuePartitionRequest) GetNamespaceId() string { return "" } -func (x *ForceLoadTaskQueuePartitionRequest) GetTaskQueuePartition() *v18.TaskQueuePartition { +func (x *ForceLoadTaskQueuePartitionRequest) GetTaskQueuePartition() *v19.TaskQueuePartition { if x != nil { return x.TaskQueuePartition } @@ -3440,7 +3463,7 @@ type ForceUnloadTaskQueueRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - TaskQueueType v19.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` + TaskQueueType v110.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3489,11 +3512,11 @@ func (x *ForceUnloadTaskQueueRequest) GetTaskQueue() string { return "" } -func (x *ForceUnloadTaskQueueRequest) GetTaskQueueType() v19.TaskQueueType { +func (x *ForceUnloadTaskQueueRequest) GetTaskQueueType() v110.TaskQueueType { if x != nil { return x.TaskQueueType } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } // TODO Shivam - Please remove this in 123 @@ -3544,7 +3567,7 @@ func (x *ForceUnloadTaskQueueResponse) GetWasLoaded() bool { type ForceUnloadTaskQueuePartitionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - TaskQueuePartition *v18.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` + TaskQueuePartition *v19.TaskQueuePartition `protobuf:"bytes,2,opt,name=task_queue_partition,json=taskQueuePartition,proto3" json:"task_queue_partition,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3586,7 +3609,7 @@ func (x *ForceUnloadTaskQueuePartitionRequest) GetNamespaceId() string { return "" } -func (x *ForceUnloadTaskQueuePartitionRequest) GetTaskQueuePartition() *v18.TaskQueuePartition { +func (x *ForceUnloadTaskQueuePartitionRequest) GetTaskQueuePartition() *v19.TaskQueuePartition { if x != nil { return x.TaskQueuePartition } @@ -3650,7 +3673,7 @@ type UpdateTaskQueueUserDataRequest struct { TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` // Versioned user data, set if the task queue has user data and the request's last_known_user_data_version is less // than the version cached in the root partition. - UserData *v111.VersionedTaskQueueUserData `protobuf:"bytes,3,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` + UserData *v112.VersionedTaskQueueUserData `protobuf:"bytes,3,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` // List of added build ids BuildIdsAdded []string `protobuf:"bytes,4,rep,name=build_ids_added,json=buildIdsAdded,proto3" json:"build_ids_added,omitempty"` // List of removed build ids @@ -3703,7 +3726,7 @@ func (x *UpdateTaskQueueUserDataRequest) GetTaskQueue() string { return "" } -func (x *UpdateTaskQueueUserDataRequest) GetUserData() *v111.VersionedTaskQueueUserData { +func (x *UpdateTaskQueueUserDataRequest) GetUserData() *v112.VersionedTaskQueueUserData { if x != nil { return x.UserData } @@ -3764,7 +3787,7 @@ type ReplicateTaskQueueUserDataRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - UserData *v111.TaskQueueUserData `protobuf:"bytes,3,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` + UserData *v112.TaskQueueUserData `protobuf:"bytes,3,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3813,7 +3836,7 @@ func (x *ReplicateTaskQueueUserDataRequest) GetTaskQueue() string { return "" } -func (x *ReplicateTaskQueueUserDataRequest) GetUserData() *v111.TaskQueueUserData { +func (x *ReplicateTaskQueueUserDataRequest) GetUserData() *v112.TaskQueueUserData { if x != nil { return x.UserData } @@ -3957,8 +3980,8 @@ type DispatchNexusTaskRequest struct { NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` TaskQueue *v14.TaskQueue `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` // Nexus request extracted by the frontend and translated into Temporal API format. - Request *v113.Request `protobuf:"bytes,3,opt,name=request,proto3" json:"request,omitempty"` - ForwardInfo *v18.TaskForwardInfo `protobuf:"bytes,4,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` + Request *v114.Request `protobuf:"bytes,3,opt,name=request,proto3" json:"request,omitempty"` + ForwardInfo *v19.TaskForwardInfo `protobuf:"bytes,4,opt,name=forward_info,json=forwardInfo,proto3" json:"forward_info,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4007,14 +4030,14 @@ func (x *DispatchNexusTaskRequest) GetTaskQueue() *v14.TaskQueue { return nil } -func (x *DispatchNexusTaskRequest) GetRequest() *v113.Request { +func (x *DispatchNexusTaskRequest) GetRequest() *v114.Request { if x != nil { return x.Request } return nil } -func (x *DispatchNexusTaskRequest) GetForwardInfo() *v18.TaskForwardInfo { +func (x *DispatchNexusTaskRequest) GetForwardInfo() *v19.TaskForwardInfo { if x != nil { return x.ForwardInfo } @@ -4072,7 +4095,7 @@ func (x *DispatchNexusTaskResponse) GetOutcome() isDispatchNexusTaskResponse_Out } // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. -func (x *DispatchNexusTaskResponse) GetHandlerError() *v113.HandlerError { +func (x *DispatchNexusTaskResponse) GetHandlerError() *v114.HandlerError { if x != nil { if x, ok := x.Outcome.(*DispatchNexusTaskResponse_HandlerError); ok { return x.HandlerError @@ -4081,7 +4104,7 @@ func (x *DispatchNexusTaskResponse) GetHandlerError() *v113.HandlerError { return nil } -func (x *DispatchNexusTaskResponse) GetResponse() *v113.Response { +func (x *DispatchNexusTaskResponse) GetResponse() *v114.Response { if x != nil { if x, ok := x.Outcome.(*DispatchNexusTaskResponse_Response); ok { return x.Response @@ -4099,7 +4122,7 @@ func (x *DispatchNexusTaskResponse) GetRequestTimeout() *DispatchNexusTaskRespon return nil } -func (x *DispatchNexusTaskResponse) GetFailure() *v114.Failure { +func (x *DispatchNexusTaskResponse) GetFailure() *v115.Failure { if x != nil { if x, ok := x.Outcome.(*DispatchNexusTaskResponse_Failure); ok { return x.Failure @@ -4116,12 +4139,12 @@ type DispatchNexusTaskResponse_HandlerError struct { // Deprecated. Use failure field instead. // // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. - HandlerError *v113.HandlerError `protobuf:"bytes,1,opt,name=handler_error,json=handlerError,proto3,oneof"` + HandlerError *v114.HandlerError `protobuf:"bytes,1,opt,name=handler_error,json=handlerError,proto3,oneof"` } type DispatchNexusTaskResponse_Response struct { // Set if the worker's handler responded successfully to the nexus task. - Response *v113.Response `protobuf:"bytes,2,opt,name=response,proto3,oneof"` + Response *v114.Response `protobuf:"bytes,2,opt,name=response,proto3,oneof"` } type DispatchNexusTaskResponse_RequestTimeout struct { @@ -4130,7 +4153,7 @@ type DispatchNexusTaskResponse_RequestTimeout struct { type DispatchNexusTaskResponse_Failure struct { // Set if the worker's handler failed the nexus task. Must contain a NexusHandlerFailureInfo object. - Failure *v114.Failure `protobuf:"bytes,4,opt,name=failure,proto3,oneof"` + Failure *v115.Failure `protobuf:"bytes,4,opt,name=failure,proto3,oneof"` } func (*DispatchNexusTaskResponse_HandlerError) isDispatchNexusTaskResponse_Outcome() {} @@ -4491,7 +4514,7 @@ func (*RespondNexusTaskFailedResponse) Descriptor() ([]byte, []int) { // aip.dev/not-precedent: CreateNexusEndpoint RPC doesn't follow Google API format. --) type CreateNexusEndpointRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Spec *v111.NexusEndpointSpec `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` + Spec *v112.NexusEndpointSpec `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4526,7 +4549,7 @@ func (*CreateNexusEndpointRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_matchingservice_v1_request_response_proto_rawDescGZIP(), []int{63} } -func (x *CreateNexusEndpointRequest) GetSpec() *v111.NexusEndpointSpec { +func (x *CreateNexusEndpointRequest) GetSpec() *v112.NexusEndpointSpec { if x != nil { return x.Spec } @@ -4535,7 +4558,7 @@ func (x *CreateNexusEndpointRequest) GetSpec() *v111.NexusEndpointSpec { type CreateNexusEndpointResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Entry *v111.NexusEndpointEntry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` + Entry *v112.NexusEndpointEntry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4570,7 +4593,7 @@ func (*CreateNexusEndpointResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_matchingservice_v1_request_response_proto_rawDescGZIP(), []int{64} } -func (x *CreateNexusEndpointResponse) GetEntry() *v111.NexusEndpointEntry { +func (x *CreateNexusEndpointResponse) GetEntry() *v112.NexusEndpointEntry { if x != nil { return x.Entry } @@ -4591,7 +4614,7 @@ type UpdateNexusEndpointRequest struct { // Version of the endpoint, used for optimistic concurrency. Must match current version in persistence or the // request will fail a FAILED_PRECONDITION error. Version int64 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"` - Spec *v111.NexusEndpointSpec `protobuf:"bytes,3,opt,name=spec,proto3" json:"spec,omitempty"` + Spec *v112.NexusEndpointSpec `protobuf:"bytes,3,opt,name=spec,proto3" json:"spec,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4640,7 +4663,7 @@ func (x *UpdateNexusEndpointRequest) GetVersion() int64 { return 0 } -func (x *UpdateNexusEndpointRequest) GetSpec() *v111.NexusEndpointSpec { +func (x *UpdateNexusEndpointRequest) GetSpec() *v112.NexusEndpointSpec { if x != nil { return x.Spec } @@ -4649,7 +4672,7 @@ func (x *UpdateNexusEndpointRequest) GetSpec() *v111.NexusEndpointSpec { type UpdateNexusEndpointResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Entry *v111.NexusEndpointEntry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` + Entry *v112.NexusEndpointEntry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4684,7 +4707,7 @@ func (*UpdateNexusEndpointResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_matchingservice_v1_request_response_proto_rawDescGZIP(), []int{66} } -func (x *UpdateNexusEndpointResponse) GetEntry() *v111.NexusEndpointEntry { +func (x *UpdateNexusEndpointResponse) GetEntry() *v112.NexusEndpointEntry { if x != nil { return x.Entry } @@ -4863,7 +4886,7 @@ type ListNexusEndpointsResponse struct { // Token for getting the next page. NextPageToken []byte `protobuf:"bytes,1,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` TableVersion int64 `protobuf:"varint,2,opt,name=table_version,json=tableVersion,proto3" json:"table_version,omitempty"` - Entries []*v111.NexusEndpointEntry `protobuf:"bytes,3,rep,name=entries,proto3" json:"entries,omitempty"` + Entries []*v112.NexusEndpointEntry `protobuf:"bytes,3,rep,name=entries,proto3" json:"entries,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4912,7 +4935,7 @@ func (x *ListNexusEndpointsResponse) GetTableVersion() int64 { return 0 } -func (x *ListNexusEndpointsResponse) GetEntries() []*v111.NexusEndpointEntry { +func (x *ListNexusEndpointsResponse) GetEntries() []*v112.NexusEndpointEntry { if x != nil { return x.Entries } @@ -5065,9 +5088,9 @@ type ListWorkersResponse struct { // includes expensive runtime metrics. We will stop populating this field in the future. // // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. - WorkersInfo []*v115.WorkerInfo `protobuf:"bytes,1,rep,name=workers_info,json=workersInfo,proto3" json:"workers_info,omitempty"` + WorkersInfo []*v116.WorkerInfo `protobuf:"bytes,1,rep,name=workers_info,json=workersInfo,proto3" json:"workers_info,omitempty"` NextPageToken []byte `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` - Workers []*v115.WorkerListInfo `protobuf:"bytes,3,rep,name=workers,proto3" json:"workers,omitempty"` + Workers []*v116.WorkerListInfo `protobuf:"bytes,3,rep,name=workers,proto3" json:"workers,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5103,7 +5126,7 @@ func (*ListWorkersResponse) Descriptor() ([]byte, []int) { } // Deprecated: Marked as deprecated in temporal/server/api/matchingservice/v1/request_response.proto. -func (x *ListWorkersResponse) GetWorkersInfo() []*v115.WorkerInfo { +func (x *ListWorkersResponse) GetWorkersInfo() []*v116.WorkerInfo { if x != nil { return x.WorkersInfo } @@ -5117,7 +5140,7 @@ func (x *ListWorkersResponse) GetNextPageToken() []byte { return nil } -func (x *ListWorkersResponse) GetWorkers() []*v115.WorkerListInfo { +func (x *ListWorkersResponse) GetWorkers() []*v116.WorkerListInfo { if x != nil { return x.Workers } @@ -5382,7 +5405,7 @@ func (x *DescribeWorkerRequest) GetRequest() *v1.DescribeWorkerRequest { type DescribeWorkerResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - WorkerInfo *v115.WorkerInfo `protobuf:"bytes,1,opt,name=worker_info,json=workerInfo,proto3" json:"worker_info,omitempty"` + WorkerInfo *v116.WorkerInfo `protobuf:"bytes,1,opt,name=worker_info,json=workerInfo,proto3" json:"worker_info,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5417,7 +5440,7 @@ func (*DescribeWorkerResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_matchingservice_v1_request_response_proto_rawDescGZIP(), []int{80} } -func (x *DescribeWorkerResponse) GetWorkerInfo() *v115.WorkerInfo { +func (x *DescribeWorkerResponse) GetWorkerInfo() *v116.WorkerInfo { if x != nil { return x.WorkerInfo } @@ -5439,8 +5462,8 @@ type UpdateFairnessStateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - TaskQueueType v19.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` - FairnessState v116.FairnessState `protobuf:"varint,4,opt,name=fairness_state,json=fairnessState,proto3,enum=temporal.server.api.enums.v1.FairnessState" json:"fairness_state,omitempty"` + TaskQueueType v110.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` + FairnessState v117.FairnessState `protobuf:"varint,4,opt,name=fairness_state,json=fairnessState,proto3,enum=temporal.server.api.enums.v1.FairnessState" json:"fairness_state,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5489,18 +5512,18 @@ func (x *UpdateFairnessStateRequest) GetTaskQueue() string { return "" } -func (x *UpdateFairnessStateRequest) GetTaskQueueType() v19.TaskQueueType { +func (x *UpdateFairnessStateRequest) GetTaskQueueType() v110.TaskQueueType { if x != nil { return x.TaskQueueType } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } -func (x *UpdateFairnessStateRequest) GetFairnessState() v116.FairnessState { +func (x *UpdateFairnessStateRequest) GetFairnessState() v117.FairnessState { if x != nil { return x.FairnessState } - return v116.FairnessState(0) + return v117.FairnessState(0) } type UpdateFairnessStateResponse struct { @@ -5543,8 +5566,8 @@ type CheckTaskQueueVersionMembershipRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` TaskQueue string `protobuf:"bytes,2,opt,name=task_queue,json=taskQueue,proto3" json:"task_queue,omitempty"` - TaskQueueType v19.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` - Version *v110.WorkerDeploymentVersion `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` + TaskQueueType v110.TaskQueueType `protobuf:"varint,3,opt,name=task_queue_type,json=taskQueueType,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"task_queue_type,omitempty"` + Version *v111.WorkerDeploymentVersion `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5593,14 +5616,14 @@ func (x *CheckTaskQueueVersionMembershipRequest) GetTaskQueue() string { return "" } -func (x *CheckTaskQueueVersionMembershipRequest) GetTaskQueueType() v19.TaskQueueType { +func (x *CheckTaskQueueVersionMembershipRequest) GetTaskQueueType() v110.TaskQueueType { if x != nil { return x.TaskQueueType } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } -func (x *CheckTaskQueueVersionMembershipRequest) GetVersion() *v110.WorkerDeploymentVersion { +func (x *CheckTaskQueueVersionMembershipRequest) GetVersion() *v111.WorkerDeploymentVersion { if x != nil { return x.Version } @@ -5799,7 +5822,7 @@ func (x *CancelOutstandingWorkerPollsPartitionRequest_WorkerEntry) GetWorkerIden type DescribeVersionedTaskQueuesRequest_VersionTaskQueue struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Type v19.TaskQueueType `protobuf:"varint,2,opt,name=type,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"type,omitempty"` + Type v110.TaskQueueType `protobuf:"varint,2,opt,name=type,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"type,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5841,18 +5864,18 @@ func (x *DescribeVersionedTaskQueuesRequest_VersionTaskQueue) GetName() string { return "" } -func (x *DescribeVersionedTaskQueuesRequest_VersionTaskQueue) GetType() v19.TaskQueueType { +func (x *DescribeVersionedTaskQueuesRequest_VersionTaskQueue) GetType() v110.TaskQueueType { if x != nil { return x.Type } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } // (-- api-linter: core::0123::resource-annotation=disabled --) type DescribeVersionedTaskQueuesResponse_VersionTaskQueue struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Type v19.TaskQueueType `protobuf:"varint,2,opt,name=type,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"type,omitempty"` + Type v110.TaskQueueType `protobuf:"varint,2,opt,name=type,proto3,enum=temporal.api.enums.v1.TaskQueueType" json:"type,omitempty"` Stats *v14.TaskQueueStats `protobuf:"bytes,3,opt,name=stats,proto3" json:"stats,omitempty"` // (-- api-linter: core::0140::prepositions=disabled // @@ -5899,11 +5922,11 @@ func (x *DescribeVersionedTaskQueuesResponse_VersionTaskQueue) GetName() string return "" } -func (x *DescribeVersionedTaskQueuesResponse_VersionTaskQueue) GetType() v19.TaskQueueType { +func (x *DescribeVersionedTaskQueuesResponse_VersionTaskQueue) GetType() v110.TaskQueueType { if x != nil { return x.Type } - return v19.TaskQueueType(0) + return v110.TaskQueueType(0) } func (x *DescribeVersionedTaskQueuesResponse_VersionTaskQueue) GetStats() *v14.TaskQueueStats { @@ -6060,7 +6083,7 @@ var File_temporal_server_api_matchingservice_v1_request_response_proto protorefl const file_temporal_server_api_matchingservice_v1_request_response_proto_rawDesc = "" + "\n" + - "=temporal/server/api/matchingservice/v1/request_response.proto\x12&temporal.server.api.matchingservice.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/deployment/v1/message.proto\x1a&temporal/api/enums/v1/task_queue.proto\x1a%temporal/api/failure/v1/message.proto\x1a%temporal/api/history/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a&temporal/api/protocol/v1/message.proto\x1a#temporal/api/query/v1/message.proto\x1a'temporal/api/taskqueue/v1/message.proto\x1a$temporal/api/worker/v1/message.proto\x1a6temporal/api/workflowservice/v1/request_response.proto\x1a*temporal/server/api/clock/v1/message.proto\x1a/temporal/server/api/deployment/v1/message.proto\x1a1temporal/server/api/enums/v1/fairness_state.proto\x1a,temporal/server/api/history/v1/message.proto\x1a.temporal/server/api/persistence/v1/nexus.proto\x1a4temporal/server/api/persistence/v1/task_queues.proto\x1a.temporal/server/api/taskqueue/v1/message.proto\"\xc3\x02\n" + + "=temporal/server/api/matchingservice/v1/request_response.proto\x12&temporal.server.api.matchingservice.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/deployment/v1/message.proto\x1a&temporal/api/enums/v1/task_queue.proto\x1a%temporal/api/failure/v1/message.proto\x1a%temporal/api/history/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a&temporal/api/protocol/v1/message.proto\x1a#temporal/api/query/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a'temporal/api/taskqueue/v1/message.proto\x1a$temporal/api/worker/v1/message.proto\x1a6temporal/api/workflowservice/v1/request_response.proto\x1a*temporal/server/api/clock/v1/message.proto\x1a/temporal/server/api/deployment/v1/message.proto\x1a1temporal/server/api/enums/v1/fairness_state.proto\x1a,temporal/server/api/history/v1/message.proto\x1a.temporal/server/api/persistence/v1/nexus.proto\x1a4temporal/server/api/persistence/v1/task_queues.proto\x1a.temporal/server/api/taskqueue/v1/message.proto\"\xc3\x02\n" + "\x1cPollWorkflowTaskQueueRequest\x12!\n" + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12\x1b\n" + "\tpoller_id\x18\x02 \x01(\tR\bpollerId\x12`\n" + @@ -6068,7 +6091,7 @@ const file_temporal_server_api_matchingservice_v1_request_response_proto_rawDesc "\x10forwarded_source\x18\x04 \x01(\tR\x0fforwardedSource\x12V\n" + "\n" + "conditions\x18\x05 \x01(\v26.temporal.server.api.matchingservice.v1.PollConditionsR\n" + - "conditions\"\xd1\v\n" + + "conditions\"\x9b\f\n" + "\x1dPollWorkflowTaskQueueResponse\x12\x1d\n" + "\n" + "task_token\x18\x01 \x01(\fR\ttaskToken\x12X\n" + @@ -6093,10 +6116,11 @@ const file_temporal_server_api_matchingservice_v1_request_response_proto_rawDesc "\x0fnext_page_token\x18\x14 \x01(\fR\rnextPageToken\x12h\n" + "\x17poller_scaling_decision\x18\x15 \x01(\v20.temporal.api.taskqueue.v1.PollerScalingDecisionR\x15pollerScalingDecision\x12A\n" + "\vraw_history\x18\x16 \x01(\v2 .temporal.api.history.v1.HistoryR\n" + - "rawHistory\x1a`\n" + + "rawHistory\x12H\n" + + "\rstream_slices\x18\x17 \x03(\v2#.temporal.api.stream.v1.StreamSliceR\fstreamSlices\x1a`\n" + "\fQueriesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12:\n" + - "\x05value\x18\x02 \x01(\v2$.temporal.api.query.v1.WorkflowQueryR\x05value:\x028\x01J\x04\b\r\x10\x0e\"\xcb\v\n" + + "\x05value\x18\x02 \x01(\v2$.temporal.api.query.v1.WorkflowQueryR\x05value:\x028\x01J\x04\b\r\x10\x0e\"\x95\f\n" + "+PollWorkflowTaskQueueResponseWithRawHistory\x12\x1d\n" + "\n" + "task_token\x18\x01 \x01(\fR\ttaskToken\x12X\n" + @@ -6121,7 +6145,8 @@ const file_temporal_server_api_matchingservice_v1_request_response_proto_rawDesc "\x0fnext_page_token\x18\x14 \x01(\fR\rnextPageToken\x12h\n" + "\x17poller_scaling_decision\x18\x15 \x01(\v20.temporal.api.taskqueue.v1.PollerScalingDecisionR\x15pollerScalingDecision\x12\x1f\n" + "\vraw_history\x18\x16 \x03(\fR\n" + - "rawHistory\x1a`\n" + + "rawHistory\x12H\n" + + "\rstream_slices\x18\x17 \x03(\v2#.temporal.api.stream.v1.StreamSliceR\fstreamSlices\x1a`\n" + "\fQueriesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12:\n" + "\x05value\x18\x02 \x01(\v2$.temporal.api.query.v1.WorkflowQueryR\x05value:\x028\x01J\x04\b\r\x10\x0e\"\xc3\x02\n" + @@ -6629,61 +6654,62 @@ var file_temporal_server_api_matchingservice_v1_request_response_proto_goTypes = (*v15.Message)(nil), // 104: temporal.api.protocol.v1.Message (*v16.History)(nil), // 105: temporal.api.history.v1.History (*v14.PollerScalingDecision)(nil), // 106: temporal.api.taskqueue.v1.PollerScalingDecision - (*v1.PollActivityTaskQueueRequest)(nil), // 107: temporal.api.workflowservice.v1.PollActivityTaskQueueRequest - (*v11.ActivityType)(nil), // 108: temporal.api.common.v1.ActivityType - (*v11.Payloads)(nil), // 109: temporal.api.common.v1.Payloads - (*durationpb.Duration)(nil), // 110: google.protobuf.Duration - (*v11.Header)(nil), // 111: temporal.api.common.v1.Header - (*v11.Priority)(nil), // 112: temporal.api.common.v1.Priority - (*v11.RetryPolicy)(nil), // 113: temporal.api.common.v1.RetryPolicy - (*v17.VectorClock)(nil), // 114: temporal.server.api.clock.v1.VectorClock - (*v18.TaskVersionDirective)(nil), // 115: temporal.server.api.taskqueue.v1.TaskVersionDirective - (*v18.TaskForwardInfo)(nil), // 116: temporal.server.api.taskqueue.v1.TaskForwardInfo - (*v1.QueryWorkflowRequest)(nil), // 117: temporal.api.workflowservice.v1.QueryWorkflowRequest - (*v12.QueryRejected)(nil), // 118: temporal.api.query.v1.QueryRejected - (*v1.RespondQueryTaskCompletedRequest)(nil), // 119: temporal.api.workflowservice.v1.RespondQueryTaskCompletedRequest - (v19.TaskQueueType)(0), // 120: temporal.api.enums.v1.TaskQueueType - (*v18.TaskQueuePartition)(nil), // 121: temporal.server.api.taskqueue.v1.TaskQueuePartition - (*v1.DescribeTaskQueueRequest)(nil), // 122: temporal.api.workflowservice.v1.DescribeTaskQueueRequest - (*v110.WorkerDeploymentVersion)(nil), // 123: temporal.server.api.deployment.v1.WorkerDeploymentVersion - (*v1.DescribeTaskQueueResponse)(nil), // 124: temporal.api.workflowservice.v1.DescribeTaskQueueResponse - (*v14.TaskQueueVersionSelection)(nil), // 125: temporal.api.taskqueue.v1.TaskQueueVersionSelection - (*v18.PartitionScaleInfo)(nil), // 126: temporal.server.api.taskqueue.v1.PartitionScaleInfo - (*v14.TaskQueuePartitionMetadata)(nil), // 127: temporal.api.taskqueue.v1.TaskQueuePartitionMetadata - (*v1.GetWorkerVersioningRulesRequest)(nil), // 128: temporal.api.workflowservice.v1.GetWorkerVersioningRulesRequest - (*v1.GetWorkerVersioningRulesResponse)(nil), // 129: temporal.api.workflowservice.v1.GetWorkerVersioningRulesResponse - (*v1.UpdateWorkerVersioningRulesRequest)(nil), // 130: temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest - (*v1.UpdateWorkerVersioningRulesResponse)(nil), // 131: temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesResponse - (*v1.GetWorkerBuildIdCompatibilityRequest)(nil), // 132: temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityRequest - (*v1.GetWorkerBuildIdCompatibilityResponse)(nil), // 133: temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityResponse - (*v111.VersionedTaskQueueUserData)(nil), // 134: temporal.server.api.persistence.v1.VersionedTaskQueueUserData - (*v18.VersionedEphemeralData)(nil), // 135: temporal.server.api.taskqueue.v1.VersionedEphemeralData - (*v110.DeploymentVersionData)(nil), // 136: temporal.server.api.deployment.v1.DeploymentVersionData - (*v112.RoutingConfig)(nil), // 137: temporal.api.deployment.v1.RoutingConfig - (*v111.TaskQueueUserData)(nil), // 138: temporal.server.api.persistence.v1.TaskQueueUserData - (*v113.Request)(nil), // 139: temporal.api.nexus.v1.Request - (*v113.HandlerError)(nil), // 140: temporal.api.nexus.v1.HandlerError - (*v113.Response)(nil), // 141: temporal.api.nexus.v1.Response - (*v114.Failure)(nil), // 142: temporal.api.failure.v1.Failure - (*v1.PollNexusTaskQueueRequest)(nil), // 143: temporal.api.workflowservice.v1.PollNexusTaskQueueRequest - (*v1.PollNexusTaskQueueResponse)(nil), // 144: temporal.api.workflowservice.v1.PollNexusTaskQueueResponse - (*v1.RespondNexusTaskCompletedRequest)(nil), // 145: temporal.api.workflowservice.v1.RespondNexusTaskCompletedRequest - (*v1.RespondNexusTaskFailedRequest)(nil), // 146: temporal.api.workflowservice.v1.RespondNexusTaskFailedRequest - (*v111.NexusEndpointSpec)(nil), // 147: temporal.server.api.persistence.v1.NexusEndpointSpec - (*v111.NexusEndpointEntry)(nil), // 148: temporal.server.api.persistence.v1.NexusEndpointEntry - (*v1.RecordWorkerHeartbeatRequest)(nil), // 149: temporal.api.workflowservice.v1.RecordWorkerHeartbeatRequest - (*v1.ListWorkersRequest)(nil), // 150: temporal.api.workflowservice.v1.ListWorkersRequest - (*v115.WorkerInfo)(nil), // 151: temporal.api.worker.v1.WorkerInfo - (*v115.WorkerListInfo)(nil), // 152: temporal.api.worker.v1.WorkerListInfo - (*v1.CountWorkersRequest)(nil), // 153: temporal.api.workflowservice.v1.CountWorkersRequest - (*v1.UpdateTaskQueueConfigRequest)(nil), // 154: temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest - (*v14.TaskQueueConfig)(nil), // 155: temporal.api.taskqueue.v1.TaskQueueConfig - (*v1.DescribeWorkerRequest)(nil), // 156: temporal.api.workflowservice.v1.DescribeWorkerRequest - (v116.FairnessState)(0), // 157: temporal.server.api.enums.v1.FairnessState - (*v14.TaskQueueStats)(nil), // 158: temporal.api.taskqueue.v1.TaskQueueStats - (*v18.TaskQueueVersionInfoInternal)(nil), // 159: temporal.server.api.taskqueue.v1.TaskQueueVersionInfoInternal - (*v1.UpdateWorkerBuildIdCompatibilityRequest)(nil), // 160: temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest - (*v110.WorkerDeploymentVersionData)(nil), // 161: temporal.server.api.deployment.v1.WorkerDeploymentVersionData + (*v17.StreamSlice)(nil), // 107: temporal.api.stream.v1.StreamSlice + (*v1.PollActivityTaskQueueRequest)(nil), // 108: temporal.api.workflowservice.v1.PollActivityTaskQueueRequest + (*v11.ActivityType)(nil), // 109: temporal.api.common.v1.ActivityType + (*v11.Payloads)(nil), // 110: temporal.api.common.v1.Payloads + (*durationpb.Duration)(nil), // 111: google.protobuf.Duration + (*v11.Header)(nil), // 112: temporal.api.common.v1.Header + (*v11.Priority)(nil), // 113: temporal.api.common.v1.Priority + (*v11.RetryPolicy)(nil), // 114: temporal.api.common.v1.RetryPolicy + (*v18.VectorClock)(nil), // 115: temporal.server.api.clock.v1.VectorClock + (*v19.TaskVersionDirective)(nil), // 116: temporal.server.api.taskqueue.v1.TaskVersionDirective + (*v19.TaskForwardInfo)(nil), // 117: temporal.server.api.taskqueue.v1.TaskForwardInfo + (*v1.QueryWorkflowRequest)(nil), // 118: temporal.api.workflowservice.v1.QueryWorkflowRequest + (*v12.QueryRejected)(nil), // 119: temporal.api.query.v1.QueryRejected + (*v1.RespondQueryTaskCompletedRequest)(nil), // 120: temporal.api.workflowservice.v1.RespondQueryTaskCompletedRequest + (v110.TaskQueueType)(0), // 121: temporal.api.enums.v1.TaskQueueType + (*v19.TaskQueuePartition)(nil), // 122: temporal.server.api.taskqueue.v1.TaskQueuePartition + (*v1.DescribeTaskQueueRequest)(nil), // 123: temporal.api.workflowservice.v1.DescribeTaskQueueRequest + (*v111.WorkerDeploymentVersion)(nil), // 124: temporal.server.api.deployment.v1.WorkerDeploymentVersion + (*v1.DescribeTaskQueueResponse)(nil), // 125: temporal.api.workflowservice.v1.DescribeTaskQueueResponse + (*v14.TaskQueueVersionSelection)(nil), // 126: temporal.api.taskqueue.v1.TaskQueueVersionSelection + (*v19.PartitionScaleInfo)(nil), // 127: temporal.server.api.taskqueue.v1.PartitionScaleInfo + (*v14.TaskQueuePartitionMetadata)(nil), // 128: temporal.api.taskqueue.v1.TaskQueuePartitionMetadata + (*v1.GetWorkerVersioningRulesRequest)(nil), // 129: temporal.api.workflowservice.v1.GetWorkerVersioningRulesRequest + (*v1.GetWorkerVersioningRulesResponse)(nil), // 130: temporal.api.workflowservice.v1.GetWorkerVersioningRulesResponse + (*v1.UpdateWorkerVersioningRulesRequest)(nil), // 131: temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest + (*v1.UpdateWorkerVersioningRulesResponse)(nil), // 132: temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesResponse + (*v1.GetWorkerBuildIdCompatibilityRequest)(nil), // 133: temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityRequest + (*v1.GetWorkerBuildIdCompatibilityResponse)(nil), // 134: temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityResponse + (*v112.VersionedTaskQueueUserData)(nil), // 135: temporal.server.api.persistence.v1.VersionedTaskQueueUserData + (*v19.VersionedEphemeralData)(nil), // 136: temporal.server.api.taskqueue.v1.VersionedEphemeralData + (*v111.DeploymentVersionData)(nil), // 137: temporal.server.api.deployment.v1.DeploymentVersionData + (*v113.RoutingConfig)(nil), // 138: temporal.api.deployment.v1.RoutingConfig + (*v112.TaskQueueUserData)(nil), // 139: temporal.server.api.persistence.v1.TaskQueueUserData + (*v114.Request)(nil), // 140: temporal.api.nexus.v1.Request + (*v114.HandlerError)(nil), // 141: temporal.api.nexus.v1.HandlerError + (*v114.Response)(nil), // 142: temporal.api.nexus.v1.Response + (*v115.Failure)(nil), // 143: temporal.api.failure.v1.Failure + (*v1.PollNexusTaskQueueRequest)(nil), // 144: temporal.api.workflowservice.v1.PollNexusTaskQueueRequest + (*v1.PollNexusTaskQueueResponse)(nil), // 145: temporal.api.workflowservice.v1.PollNexusTaskQueueResponse + (*v1.RespondNexusTaskCompletedRequest)(nil), // 146: temporal.api.workflowservice.v1.RespondNexusTaskCompletedRequest + (*v1.RespondNexusTaskFailedRequest)(nil), // 147: temporal.api.workflowservice.v1.RespondNexusTaskFailedRequest + (*v112.NexusEndpointSpec)(nil), // 148: temporal.server.api.persistence.v1.NexusEndpointSpec + (*v112.NexusEndpointEntry)(nil), // 149: temporal.server.api.persistence.v1.NexusEndpointEntry + (*v1.RecordWorkerHeartbeatRequest)(nil), // 150: temporal.api.workflowservice.v1.RecordWorkerHeartbeatRequest + (*v1.ListWorkersRequest)(nil), // 151: temporal.api.workflowservice.v1.ListWorkersRequest + (*v116.WorkerInfo)(nil), // 152: temporal.api.worker.v1.WorkerInfo + (*v116.WorkerListInfo)(nil), // 153: temporal.api.worker.v1.WorkerListInfo + (*v1.CountWorkersRequest)(nil), // 154: temporal.api.workflowservice.v1.CountWorkersRequest + (*v1.UpdateTaskQueueConfigRequest)(nil), // 155: temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest + (*v14.TaskQueueConfig)(nil), // 156: temporal.api.taskqueue.v1.TaskQueueConfig + (*v1.DescribeWorkerRequest)(nil), // 157: temporal.api.workflowservice.v1.DescribeWorkerRequest + (v117.FairnessState)(0), // 158: temporal.server.api.enums.v1.FairnessState + (*v14.TaskQueueStats)(nil), // 159: temporal.api.taskqueue.v1.TaskQueueStats + (*v19.TaskQueueVersionInfoInternal)(nil), // 160: temporal.server.api.taskqueue.v1.TaskQueueVersionInfoInternal + (*v1.UpdateWorkerBuildIdCompatibilityRequest)(nil), // 161: temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest + (*v111.WorkerDeploymentVersionData)(nil), // 162: temporal.server.api.deployment.v1.WorkerDeploymentVersionData } var file_temporal_server_api_matchingservice_v1_request_response_proto_depIdxs = []int32{ 97, // 0: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueRequest @@ -6700,148 +6726,150 @@ var file_temporal_server_api_matchingservice_v1_request_response_proto_depIdxs = 105, // 11: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponse.history:type_name -> temporal.api.history.v1.History 106, // 12: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponse.poller_scaling_decision:type_name -> temporal.api.taskqueue.v1.PollerScalingDecision 105, // 13: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponse.raw_history:type_name -> temporal.api.history.v1.History - 98, // 14: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 99, // 15: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 100, // 16: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.query:type_name -> temporal.api.query.v1.WorkflowQuery - 101, // 17: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo - 102, // 18: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 103, // 19: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.scheduled_time:type_name -> google.protobuf.Timestamp - 103, // 20: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.started_time:type_name -> google.protobuf.Timestamp - 87, // 21: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.queries:type_name -> temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.QueriesEntry - 104, // 22: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.messages:type_name -> temporal.api.protocol.v1.Message - 105, // 23: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.history:type_name -> temporal.api.history.v1.History - 106, // 24: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.poller_scaling_decision:type_name -> temporal.api.taskqueue.v1.PollerScalingDecision - 107, // 25: temporal.server.api.matchingservice.v1.PollActivityTaskQueueRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueRequest - 85, // 26: temporal.server.api.matchingservice.v1.PollActivityTaskQueueRequest.conditions:type_name -> temporal.server.api.matchingservice.v1.PollConditions - 98, // 27: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 108, // 28: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.activity_type:type_name -> temporal.api.common.v1.ActivityType - 109, // 29: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.input:type_name -> temporal.api.common.v1.Payloads - 103, // 30: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.scheduled_time:type_name -> google.protobuf.Timestamp - 110, // 31: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.schedule_to_close_timeout:type_name -> google.protobuf.Duration - 103, // 32: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.started_time:type_name -> google.protobuf.Timestamp - 110, // 33: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.start_to_close_timeout:type_name -> google.protobuf.Duration - 110, // 34: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.heartbeat_timeout:type_name -> google.protobuf.Duration - 103, // 35: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.current_attempt_scheduled_time:type_name -> google.protobuf.Timestamp - 109, // 36: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.heartbeat_details:type_name -> temporal.api.common.v1.Payloads - 99, // 37: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 111, // 38: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.header:type_name -> temporal.api.common.v1.Header - 106, // 39: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.poller_scaling_decision:type_name -> temporal.api.taskqueue.v1.PollerScalingDecision - 112, // 40: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.priority:type_name -> temporal.api.common.v1.Priority - 113, // 41: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.retry_policy:type_name -> temporal.api.common.v1.RetryPolicy - 98, // 42: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 102, // 43: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 110, // 44: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.schedule_to_start_timeout:type_name -> google.protobuf.Duration - 114, // 45: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 115, // 46: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective - 116, // 47: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo - 112, // 48: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.priority:type_name -> temporal.api.common.v1.Priority - 98, // 49: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 102, // 50: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 110, // 51: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.schedule_to_start_timeout:type_name -> google.protobuf.Duration - 114, // 52: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 115, // 53: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective - 116, // 54: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo - 112, // 55: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.priority:type_name -> temporal.api.common.v1.Priority - 102, // 56: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 117, // 57: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.query_request:type_name -> temporal.api.workflowservice.v1.QueryWorkflowRequest - 115, // 58: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective - 116, // 59: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo - 112, // 60: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.priority:type_name -> temporal.api.common.v1.Priority - 109, // 61: temporal.server.api.matchingservice.v1.QueryWorkflowResponse.query_result:type_name -> temporal.api.common.v1.Payloads - 118, // 62: temporal.server.api.matchingservice.v1.QueryWorkflowResponse.query_rejected:type_name -> temporal.api.query.v1.QueryRejected - 102, // 63: temporal.server.api.matchingservice.v1.RespondQueryTaskCompletedRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 119, // 64: temporal.server.api.matchingservice.v1.RespondQueryTaskCompletedRequest.completed_request:type_name -> temporal.api.workflowservice.v1.RespondQueryTaskCompletedRequest - 120, // 65: temporal.server.api.matchingservice.v1.CancelOutstandingPollRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType - 102, // 66: temporal.server.api.matchingservice.v1.CancelOutstandingPollRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 102, // 67: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 120, // 68: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType - 121, // 69: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition - 121, // 70: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.partitions:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition - 88, // 71: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.workers:type_name -> temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.WorkerEntry - 122, // 72: temporal.server.api.matchingservice.v1.DescribeTaskQueueRequest.desc_request:type_name -> temporal.api.workflowservice.v1.DescribeTaskQueueRequest - 123, // 73: temporal.server.api.matchingservice.v1.DescribeTaskQueueRequest.version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion - 124, // 74: temporal.server.api.matchingservice.v1.DescribeTaskQueueResponse.desc_response:type_name -> temporal.api.workflowservice.v1.DescribeTaskQueueResponse - 120, // 75: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType - 102, // 76: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 123, // 77: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion - 89, // 78: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.version_task_queues:type_name -> temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.VersionTaskQueue - 90, // 79: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.version_task_queues:type_name -> temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue - 121, // 80: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition - 125, // 81: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionRequest.versions:type_name -> temporal.api.taskqueue.v1.TaskQueueVersionSelection - 92, // 82: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.versions_info_internal:type_name -> temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.VersionsInfoInternalEntry - 126, // 83: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.scale_info:type_name -> temporal.server.api.taskqueue.v1.PartitionScaleInfo - 102, // 84: temporal.server.api.matchingservice.v1.ListTaskQueuePartitionsRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 127, // 85: temporal.server.api.matchingservice.v1.ListTaskQueuePartitionsResponse.activity_task_queue_partitions:type_name -> temporal.api.taskqueue.v1.TaskQueuePartitionMetadata - 127, // 86: temporal.server.api.matchingservice.v1.ListTaskQueuePartitionsResponse.workflow_task_queue_partitions:type_name -> temporal.api.taskqueue.v1.TaskQueuePartitionMetadata - 93, // 87: temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.apply_public_request:type_name -> temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.ApplyPublicRequest - 94, // 88: temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.remove_build_ids:type_name -> temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.RemoveBuildIds - 128, // 89: temporal.server.api.matchingservice.v1.GetWorkerVersioningRulesRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkerVersioningRulesRequest - 129, // 90: temporal.server.api.matchingservice.v1.GetWorkerVersioningRulesResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkerVersioningRulesResponse - 130, // 91: temporal.server.api.matchingservice.v1.UpdateWorkerVersioningRulesRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest - 131, // 92: temporal.server.api.matchingservice.v1.UpdateWorkerVersioningRulesResponse.response:type_name -> temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesResponse - 132, // 93: temporal.server.api.matchingservice.v1.GetWorkerBuildIdCompatibilityRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityRequest - 133, // 94: temporal.server.api.matchingservice.v1.GetWorkerBuildIdCompatibilityResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityResponse - 120, // 95: temporal.server.api.matchingservice.v1.GetTaskQueueUserDataRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType - 134, // 96: temporal.server.api.matchingservice.v1.GetTaskQueueUserDataResponse.user_data:type_name -> temporal.server.api.persistence.v1.VersionedTaskQueueUserData - 135, // 97: temporal.server.api.matchingservice.v1.GetTaskQueueUserDataResponse.ephemeral_data:type_name -> temporal.server.api.taskqueue.v1.VersionedEphemeralData - 120, // 98: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.task_queue_types:type_name -> temporal.api.enums.v1.TaskQueueType - 136, // 99: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.update_version_data:type_name -> temporal.server.api.deployment.v1.DeploymentVersionData - 123, // 100: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.forget_version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion - 137, // 101: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.update_routing_config:type_name -> temporal.api.deployment.v1.RoutingConfig - 95, // 102: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.upsert_versions_data:type_name -> temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.UpsertVersionsDataEntry - 138, // 103: temporal.server.api.matchingservice.v1.ApplyTaskQueueUserDataReplicationEventRequest.user_data:type_name -> temporal.server.api.persistence.v1.TaskQueueUserData - 121, // 104: temporal.server.api.matchingservice.v1.ForceLoadTaskQueuePartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition - 120, // 105: temporal.server.api.matchingservice.v1.ForceUnloadTaskQueueRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType - 121, // 106: temporal.server.api.matchingservice.v1.ForceUnloadTaskQueuePartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition - 134, // 107: temporal.server.api.matchingservice.v1.UpdateTaskQueueUserDataRequest.user_data:type_name -> temporal.server.api.persistence.v1.VersionedTaskQueueUserData - 138, // 108: temporal.server.api.matchingservice.v1.ReplicateTaskQueueUserDataRequest.user_data:type_name -> temporal.server.api.persistence.v1.TaskQueueUserData - 102, // 109: temporal.server.api.matchingservice.v1.DispatchNexusTaskRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 139, // 110: temporal.server.api.matchingservice.v1.DispatchNexusTaskRequest.request:type_name -> temporal.api.nexus.v1.Request - 116, // 111: temporal.server.api.matchingservice.v1.DispatchNexusTaskRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo - 140, // 112: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.handler_error:type_name -> temporal.api.nexus.v1.HandlerError - 141, // 113: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.response:type_name -> temporal.api.nexus.v1.Response - 96, // 114: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.request_timeout:type_name -> temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.Timeout - 142, // 115: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.failure:type_name -> temporal.api.failure.v1.Failure - 143, // 116: temporal.server.api.matchingservice.v1.PollNexusTaskQueueRequest.request:type_name -> temporal.api.workflowservice.v1.PollNexusTaskQueueRequest - 85, // 117: temporal.server.api.matchingservice.v1.PollNexusTaskQueueRequest.conditions:type_name -> temporal.server.api.matchingservice.v1.PollConditions - 144, // 118: temporal.server.api.matchingservice.v1.PollNexusTaskQueueResponse.response:type_name -> temporal.api.workflowservice.v1.PollNexusTaskQueueResponse - 102, // 119: temporal.server.api.matchingservice.v1.RespondNexusTaskCompletedRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 145, // 120: temporal.server.api.matchingservice.v1.RespondNexusTaskCompletedRequest.request:type_name -> temporal.api.workflowservice.v1.RespondNexusTaskCompletedRequest - 102, // 121: temporal.server.api.matchingservice.v1.RespondNexusTaskFailedRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 146, // 122: temporal.server.api.matchingservice.v1.RespondNexusTaskFailedRequest.request:type_name -> temporal.api.workflowservice.v1.RespondNexusTaskFailedRequest - 147, // 123: temporal.server.api.matchingservice.v1.CreateNexusEndpointRequest.spec:type_name -> temporal.server.api.persistence.v1.NexusEndpointSpec - 148, // 124: temporal.server.api.matchingservice.v1.CreateNexusEndpointResponse.entry:type_name -> temporal.server.api.persistence.v1.NexusEndpointEntry - 147, // 125: temporal.server.api.matchingservice.v1.UpdateNexusEndpointRequest.spec:type_name -> temporal.server.api.persistence.v1.NexusEndpointSpec - 148, // 126: temporal.server.api.matchingservice.v1.UpdateNexusEndpointResponse.entry:type_name -> temporal.server.api.persistence.v1.NexusEndpointEntry - 148, // 127: temporal.server.api.matchingservice.v1.ListNexusEndpointsResponse.entries:type_name -> temporal.server.api.persistence.v1.NexusEndpointEntry - 149, // 128: temporal.server.api.matchingservice.v1.RecordWorkerHeartbeatRequest.heartbeart_request:type_name -> temporal.api.workflowservice.v1.RecordWorkerHeartbeatRequest - 150, // 129: temporal.server.api.matchingservice.v1.ListWorkersRequest.list_request:type_name -> temporal.api.workflowservice.v1.ListWorkersRequest - 151, // 130: temporal.server.api.matchingservice.v1.ListWorkersResponse.workers_info:type_name -> temporal.api.worker.v1.WorkerInfo - 152, // 131: temporal.server.api.matchingservice.v1.ListWorkersResponse.workers:type_name -> temporal.api.worker.v1.WorkerListInfo - 153, // 132: temporal.server.api.matchingservice.v1.CountWorkersRequest.count_request:type_name -> temporal.api.workflowservice.v1.CountWorkersRequest - 154, // 133: temporal.server.api.matchingservice.v1.UpdateTaskQueueConfigRequest.update_taskqueue_config:type_name -> temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest - 155, // 134: temporal.server.api.matchingservice.v1.UpdateTaskQueueConfigResponse.updated_taskqueue_config:type_name -> temporal.api.taskqueue.v1.TaskQueueConfig - 156, // 135: temporal.server.api.matchingservice.v1.DescribeWorkerRequest.request:type_name -> temporal.api.workflowservice.v1.DescribeWorkerRequest - 151, // 136: temporal.server.api.matchingservice.v1.DescribeWorkerResponse.worker_info:type_name -> temporal.api.worker.v1.WorkerInfo - 120, // 137: temporal.server.api.matchingservice.v1.UpdateFairnessStateRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType - 157, // 138: temporal.server.api.matchingservice.v1.UpdateFairnessStateRequest.fairness_state:type_name -> temporal.server.api.enums.v1.FairnessState - 120, // 139: temporal.server.api.matchingservice.v1.CheckTaskQueueVersionMembershipRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType - 123, // 140: temporal.server.api.matchingservice.v1.CheckTaskQueueVersionMembershipRequest.version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion - 100, // 141: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponse.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery - 100, // 142: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery - 120, // 143: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.VersionTaskQueue.type:type_name -> temporal.api.enums.v1.TaskQueueType - 120, // 144: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.type:type_name -> temporal.api.enums.v1.TaskQueueType - 158, // 145: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.stats:type_name -> temporal.api.taskqueue.v1.TaskQueueStats - 91, // 146: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.stats_by_priority_key:type_name -> temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.StatsByPriorityKeyEntry - 158, // 147: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.StatsByPriorityKeyEntry.value:type_name -> temporal.api.taskqueue.v1.TaskQueueStats - 159, // 148: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.VersionsInfoInternalEntry.value:type_name -> temporal.server.api.taskqueue.v1.TaskQueueVersionInfoInternal - 160, // 149: temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.ApplyPublicRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest - 161, // 150: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.UpsertVersionsDataEntry.value:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersionData - 151, // [151:151] is the sub-list for method output_type - 151, // [151:151] is the sub-list for method input_type - 151, // [151:151] is the sub-list for extension type_name - 151, // [151:151] is the sub-list for extension extendee - 0, // [0:151] is the sub-list for field type_name + 107, // 14: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponse.stream_slices:type_name -> temporal.api.stream.v1.StreamSlice + 98, // 15: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 99, // 16: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 100, // 17: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.query:type_name -> temporal.api.query.v1.WorkflowQuery + 101, // 18: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo + 102, // 19: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 103, // 20: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.scheduled_time:type_name -> google.protobuf.Timestamp + 103, // 21: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.started_time:type_name -> google.protobuf.Timestamp + 87, // 22: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.queries:type_name -> temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.QueriesEntry + 104, // 23: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.messages:type_name -> temporal.api.protocol.v1.Message + 105, // 24: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.history:type_name -> temporal.api.history.v1.History + 106, // 25: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.poller_scaling_decision:type_name -> temporal.api.taskqueue.v1.PollerScalingDecision + 107, // 26: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.stream_slices:type_name -> temporal.api.stream.v1.StreamSlice + 108, // 27: temporal.server.api.matchingservice.v1.PollActivityTaskQueueRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueRequest + 85, // 28: temporal.server.api.matchingservice.v1.PollActivityTaskQueueRequest.conditions:type_name -> temporal.server.api.matchingservice.v1.PollConditions + 98, // 29: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 109, // 30: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.activity_type:type_name -> temporal.api.common.v1.ActivityType + 110, // 31: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.input:type_name -> temporal.api.common.v1.Payloads + 103, // 32: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.scheduled_time:type_name -> google.protobuf.Timestamp + 111, // 33: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.schedule_to_close_timeout:type_name -> google.protobuf.Duration + 103, // 34: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.started_time:type_name -> google.protobuf.Timestamp + 111, // 35: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.start_to_close_timeout:type_name -> google.protobuf.Duration + 111, // 36: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.heartbeat_timeout:type_name -> google.protobuf.Duration + 103, // 37: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.current_attempt_scheduled_time:type_name -> google.protobuf.Timestamp + 110, // 38: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.heartbeat_details:type_name -> temporal.api.common.v1.Payloads + 99, // 39: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 112, // 40: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.header:type_name -> temporal.api.common.v1.Header + 106, // 41: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.poller_scaling_decision:type_name -> temporal.api.taskqueue.v1.PollerScalingDecision + 113, // 42: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.priority:type_name -> temporal.api.common.v1.Priority + 114, // 43: temporal.server.api.matchingservice.v1.PollActivityTaskQueueResponse.retry_policy:type_name -> temporal.api.common.v1.RetryPolicy + 98, // 44: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 102, // 45: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 111, // 46: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.schedule_to_start_timeout:type_name -> google.protobuf.Duration + 115, // 47: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 116, // 48: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective + 117, // 49: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo + 113, // 50: temporal.server.api.matchingservice.v1.AddWorkflowTaskRequest.priority:type_name -> temporal.api.common.v1.Priority + 98, // 51: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 102, // 52: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 111, // 53: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.schedule_to_start_timeout:type_name -> google.protobuf.Duration + 115, // 54: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 116, // 55: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective + 117, // 56: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo + 113, // 57: temporal.server.api.matchingservice.v1.AddActivityTaskRequest.priority:type_name -> temporal.api.common.v1.Priority + 102, // 58: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 118, // 59: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.query_request:type_name -> temporal.api.workflowservice.v1.QueryWorkflowRequest + 116, // 60: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective + 117, // 61: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo + 113, // 62: temporal.server.api.matchingservice.v1.QueryWorkflowRequest.priority:type_name -> temporal.api.common.v1.Priority + 110, // 63: temporal.server.api.matchingservice.v1.QueryWorkflowResponse.query_result:type_name -> temporal.api.common.v1.Payloads + 119, // 64: temporal.server.api.matchingservice.v1.QueryWorkflowResponse.query_rejected:type_name -> temporal.api.query.v1.QueryRejected + 102, // 65: temporal.server.api.matchingservice.v1.RespondQueryTaskCompletedRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 120, // 66: temporal.server.api.matchingservice.v1.RespondQueryTaskCompletedRequest.completed_request:type_name -> temporal.api.workflowservice.v1.RespondQueryTaskCompletedRequest + 121, // 67: temporal.server.api.matchingservice.v1.CancelOutstandingPollRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType + 102, // 68: temporal.server.api.matchingservice.v1.CancelOutstandingPollRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 102, // 69: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 121, // 70: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType + 122, // 71: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition + 122, // 72: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.partitions:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition + 88, // 73: temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.workers:type_name -> temporal.server.api.matchingservice.v1.CancelOutstandingWorkerPollsPartitionRequest.WorkerEntry + 123, // 74: temporal.server.api.matchingservice.v1.DescribeTaskQueueRequest.desc_request:type_name -> temporal.api.workflowservice.v1.DescribeTaskQueueRequest + 124, // 75: temporal.server.api.matchingservice.v1.DescribeTaskQueueRequest.version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion + 125, // 76: temporal.server.api.matchingservice.v1.DescribeTaskQueueResponse.desc_response:type_name -> temporal.api.workflowservice.v1.DescribeTaskQueueResponse + 121, // 77: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType + 102, // 78: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 124, // 79: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion + 89, // 80: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.version_task_queues:type_name -> temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.VersionTaskQueue + 90, // 81: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.version_task_queues:type_name -> temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue + 122, // 82: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition + 126, // 83: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionRequest.versions:type_name -> temporal.api.taskqueue.v1.TaskQueueVersionSelection + 92, // 84: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.versions_info_internal:type_name -> temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.VersionsInfoInternalEntry + 127, // 85: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.scale_info:type_name -> temporal.server.api.taskqueue.v1.PartitionScaleInfo + 102, // 86: temporal.server.api.matchingservice.v1.ListTaskQueuePartitionsRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 128, // 87: temporal.server.api.matchingservice.v1.ListTaskQueuePartitionsResponse.activity_task_queue_partitions:type_name -> temporal.api.taskqueue.v1.TaskQueuePartitionMetadata + 128, // 88: temporal.server.api.matchingservice.v1.ListTaskQueuePartitionsResponse.workflow_task_queue_partitions:type_name -> temporal.api.taskqueue.v1.TaskQueuePartitionMetadata + 93, // 89: temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.apply_public_request:type_name -> temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.ApplyPublicRequest + 94, // 90: temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.remove_build_ids:type_name -> temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.RemoveBuildIds + 129, // 91: temporal.server.api.matchingservice.v1.GetWorkerVersioningRulesRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkerVersioningRulesRequest + 130, // 92: temporal.server.api.matchingservice.v1.GetWorkerVersioningRulesResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkerVersioningRulesResponse + 131, // 93: temporal.server.api.matchingservice.v1.UpdateWorkerVersioningRulesRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesRequest + 132, // 94: temporal.server.api.matchingservice.v1.UpdateWorkerVersioningRulesResponse.response:type_name -> temporal.api.workflowservice.v1.UpdateWorkerVersioningRulesResponse + 133, // 95: temporal.server.api.matchingservice.v1.GetWorkerBuildIdCompatibilityRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityRequest + 134, // 96: temporal.server.api.matchingservice.v1.GetWorkerBuildIdCompatibilityResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkerBuildIdCompatibilityResponse + 121, // 97: temporal.server.api.matchingservice.v1.GetTaskQueueUserDataRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType + 135, // 98: temporal.server.api.matchingservice.v1.GetTaskQueueUserDataResponse.user_data:type_name -> temporal.server.api.persistence.v1.VersionedTaskQueueUserData + 136, // 99: temporal.server.api.matchingservice.v1.GetTaskQueueUserDataResponse.ephemeral_data:type_name -> temporal.server.api.taskqueue.v1.VersionedEphemeralData + 121, // 100: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.task_queue_types:type_name -> temporal.api.enums.v1.TaskQueueType + 137, // 101: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.update_version_data:type_name -> temporal.server.api.deployment.v1.DeploymentVersionData + 124, // 102: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.forget_version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion + 138, // 103: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.update_routing_config:type_name -> temporal.api.deployment.v1.RoutingConfig + 95, // 104: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.upsert_versions_data:type_name -> temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.UpsertVersionsDataEntry + 139, // 105: temporal.server.api.matchingservice.v1.ApplyTaskQueueUserDataReplicationEventRequest.user_data:type_name -> temporal.server.api.persistence.v1.TaskQueueUserData + 122, // 106: temporal.server.api.matchingservice.v1.ForceLoadTaskQueuePartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition + 121, // 107: temporal.server.api.matchingservice.v1.ForceUnloadTaskQueueRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType + 122, // 108: temporal.server.api.matchingservice.v1.ForceUnloadTaskQueuePartitionRequest.task_queue_partition:type_name -> temporal.server.api.taskqueue.v1.TaskQueuePartition + 135, // 109: temporal.server.api.matchingservice.v1.UpdateTaskQueueUserDataRequest.user_data:type_name -> temporal.server.api.persistence.v1.VersionedTaskQueueUserData + 139, // 110: temporal.server.api.matchingservice.v1.ReplicateTaskQueueUserDataRequest.user_data:type_name -> temporal.server.api.persistence.v1.TaskQueueUserData + 102, // 111: temporal.server.api.matchingservice.v1.DispatchNexusTaskRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 140, // 112: temporal.server.api.matchingservice.v1.DispatchNexusTaskRequest.request:type_name -> temporal.api.nexus.v1.Request + 117, // 113: temporal.server.api.matchingservice.v1.DispatchNexusTaskRequest.forward_info:type_name -> temporal.server.api.taskqueue.v1.TaskForwardInfo + 141, // 114: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.handler_error:type_name -> temporal.api.nexus.v1.HandlerError + 142, // 115: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.response:type_name -> temporal.api.nexus.v1.Response + 96, // 116: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.request_timeout:type_name -> temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.Timeout + 143, // 117: temporal.server.api.matchingservice.v1.DispatchNexusTaskResponse.failure:type_name -> temporal.api.failure.v1.Failure + 144, // 118: temporal.server.api.matchingservice.v1.PollNexusTaskQueueRequest.request:type_name -> temporal.api.workflowservice.v1.PollNexusTaskQueueRequest + 85, // 119: temporal.server.api.matchingservice.v1.PollNexusTaskQueueRequest.conditions:type_name -> temporal.server.api.matchingservice.v1.PollConditions + 145, // 120: temporal.server.api.matchingservice.v1.PollNexusTaskQueueResponse.response:type_name -> temporal.api.workflowservice.v1.PollNexusTaskQueueResponse + 102, // 121: temporal.server.api.matchingservice.v1.RespondNexusTaskCompletedRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 146, // 122: temporal.server.api.matchingservice.v1.RespondNexusTaskCompletedRequest.request:type_name -> temporal.api.workflowservice.v1.RespondNexusTaskCompletedRequest + 102, // 123: temporal.server.api.matchingservice.v1.RespondNexusTaskFailedRequest.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 147, // 124: temporal.server.api.matchingservice.v1.RespondNexusTaskFailedRequest.request:type_name -> temporal.api.workflowservice.v1.RespondNexusTaskFailedRequest + 148, // 125: temporal.server.api.matchingservice.v1.CreateNexusEndpointRequest.spec:type_name -> temporal.server.api.persistence.v1.NexusEndpointSpec + 149, // 126: temporal.server.api.matchingservice.v1.CreateNexusEndpointResponse.entry:type_name -> temporal.server.api.persistence.v1.NexusEndpointEntry + 148, // 127: temporal.server.api.matchingservice.v1.UpdateNexusEndpointRequest.spec:type_name -> temporal.server.api.persistence.v1.NexusEndpointSpec + 149, // 128: temporal.server.api.matchingservice.v1.UpdateNexusEndpointResponse.entry:type_name -> temporal.server.api.persistence.v1.NexusEndpointEntry + 149, // 129: temporal.server.api.matchingservice.v1.ListNexusEndpointsResponse.entries:type_name -> temporal.server.api.persistence.v1.NexusEndpointEntry + 150, // 130: temporal.server.api.matchingservice.v1.RecordWorkerHeartbeatRequest.heartbeart_request:type_name -> temporal.api.workflowservice.v1.RecordWorkerHeartbeatRequest + 151, // 131: temporal.server.api.matchingservice.v1.ListWorkersRequest.list_request:type_name -> temporal.api.workflowservice.v1.ListWorkersRequest + 152, // 132: temporal.server.api.matchingservice.v1.ListWorkersResponse.workers_info:type_name -> temporal.api.worker.v1.WorkerInfo + 153, // 133: temporal.server.api.matchingservice.v1.ListWorkersResponse.workers:type_name -> temporal.api.worker.v1.WorkerListInfo + 154, // 134: temporal.server.api.matchingservice.v1.CountWorkersRequest.count_request:type_name -> temporal.api.workflowservice.v1.CountWorkersRequest + 155, // 135: temporal.server.api.matchingservice.v1.UpdateTaskQueueConfigRequest.update_taskqueue_config:type_name -> temporal.api.workflowservice.v1.UpdateTaskQueueConfigRequest + 156, // 136: temporal.server.api.matchingservice.v1.UpdateTaskQueueConfigResponse.updated_taskqueue_config:type_name -> temporal.api.taskqueue.v1.TaskQueueConfig + 157, // 137: temporal.server.api.matchingservice.v1.DescribeWorkerRequest.request:type_name -> temporal.api.workflowservice.v1.DescribeWorkerRequest + 152, // 138: temporal.server.api.matchingservice.v1.DescribeWorkerResponse.worker_info:type_name -> temporal.api.worker.v1.WorkerInfo + 121, // 139: temporal.server.api.matchingservice.v1.UpdateFairnessStateRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType + 158, // 140: temporal.server.api.matchingservice.v1.UpdateFairnessStateRequest.fairness_state:type_name -> temporal.server.api.enums.v1.FairnessState + 121, // 141: temporal.server.api.matchingservice.v1.CheckTaskQueueVersionMembershipRequest.task_queue_type:type_name -> temporal.api.enums.v1.TaskQueueType + 124, // 142: temporal.server.api.matchingservice.v1.CheckTaskQueueVersionMembershipRequest.version:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersion + 100, // 143: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponse.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery + 100, // 144: temporal.server.api.matchingservice.v1.PollWorkflowTaskQueueResponseWithRawHistory.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery + 121, // 145: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesRequest.VersionTaskQueue.type:type_name -> temporal.api.enums.v1.TaskQueueType + 121, // 146: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.type:type_name -> temporal.api.enums.v1.TaskQueueType + 159, // 147: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.stats:type_name -> temporal.api.taskqueue.v1.TaskQueueStats + 91, // 148: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.stats_by_priority_key:type_name -> temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.StatsByPriorityKeyEntry + 159, // 149: temporal.server.api.matchingservice.v1.DescribeVersionedTaskQueuesResponse.VersionTaskQueue.StatsByPriorityKeyEntry.value:type_name -> temporal.api.taskqueue.v1.TaskQueueStats + 160, // 150: temporal.server.api.matchingservice.v1.DescribeTaskQueuePartitionResponse.VersionsInfoInternalEntry.value:type_name -> temporal.server.api.taskqueue.v1.TaskQueueVersionInfoInternal + 161, // 151: temporal.server.api.matchingservice.v1.UpdateWorkerBuildIdCompatibilityRequest.ApplyPublicRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkerBuildIdCompatibilityRequest + 162, // 152: temporal.server.api.matchingservice.v1.SyncDeploymentUserDataRequest.UpsertVersionsDataEntry.value:type_name -> temporal.server.api.deployment.v1.WorkerDeploymentVersionData + 153, // [153:153] is the sub-list for method output_type + 153, // [153:153] is the sub-list for method input_type + 153, // [153:153] is the sub-list for extension type_name + 153, // [153:153] is the sub-list for extension extendee + 0, // [0:153] is the sub-list for field type_name } func init() { file_temporal_server_api_matchingservice_v1_request_response_proto_init() } diff --git a/chasm-external-data-request.md b/chasm-external-data-request.md new file mode 100644 index 00000000000..444296c1bed --- /dev/null +++ b/chasm-external-data-request.md @@ -0,0 +1,123 @@ +# Where a stream's bytes live: asked, and answered + +Status: **answered 2026-09-02.** This started as a request to CHASM owners for a +node kind covering data a component owns but does not store inline. The answer +is that it is already planned, so the request is withdrawn and this file is the +record of the answer and what it means for the prototype. + +## The answer + +Yichao Yang: the underlying event storage in CHASM is planned to be exposed in +**H2, with replication and lifecycle**. Roey Berman confirmed that streaming was +expected to use it. So the general capability is coming, and nothing in this +prototype should be built to substitute for it. + +That closes the question this file originally asked. Not by argument, but +because the work is scheduled and owned elsewhere. + +## Why the question came up + +A stream's messages cannot live in mutable state. All four CHASM node kinds +store their bytes inline (`chasm.proto:27-32`) in +`WorkflowMutableState.chasm_nodes` (`workflow_mutable_state.proto:19`), and: + +- `chasmNodeSizes` (`mutable_state_impl.go:170`) feeds `approximateSize` +- checked against `MutableStateSizeLimitError`, 8 MB, warn at 1 MB + (`dynamicconfig/constants.go:473-481`) +- over the error limit the execution is **force-terminated** + (`context.go:1381`, `maxMutableStateSizeExceeded`) +- archetype-tagged, so a standalone stream entity is subject to it exactly as a + workflow is + +Measured: a 100k-token stream is 5.84 MB of payload. Inline, that is one long +agent conversation past the warn and approaching the limit, and every append +rewrites the whole record on the way there. + +## Prior art, which reached the same ceiling + +Dan Davison and Sean Kane built a CHASM streaming component about a year ago, +on `seandan/streaming` in `temporalio/temporal`. It is roughly 1500 lines under +the same `chasm/lib/stream/` path this prototype uses, with `AddToStream` and +`PollStream` on the frontend and one end-to-end test. Its component is: + +```go +type Stream struct { + chasm.UnimplementedComponent + *streampb.StreamState // head, tail + Messages chasm.Map[int64, *commonpb.Payload] // one data node per message +} +``` + +Roey Berman flagged its ceiling at the time: limited to about 5 MB of total +payload because everything is in mutable state. The same note enumerated three +ways out, and they are the same three this prototype arrived at independently: +CHASM nodes in separate cells, payloads written outside mutable state by +repurposing history, or payloads replaced by pointers to a blob store. + +Worth being plain about it: this prototype's first substrate was the second of +those, and the request this file used to make was the third. Neither was a new +idea. Finding that out after the fact is the cost of not having read +`#crew-streaming` first. + +## The one thing that is not settled + +Dan Davison's guidance is to not make the storage realistic: use something based +on `chasm.Map` and assume it has the properties it needs, because the value of +this prototype is exploring user-facing behaviour that other design sessions +might miss. That is a coherent position and it is what the prior art does. + +It does not carry the benchmark, which is the other half of AI-198. The point of +measuring client-side streaming against server-side streaming is to find out +what each costs, and a substrate that force-terminates the entity partway +through the workload cannot produce a number anyone should quote. The two goals +want different things from the same prototype, and that is the disagreement to +resolve rather than paper over. + +Separately, Paul Nordstrom has asked for a discussion before this goes further, +on the grounds that the data team owns backend storage for the stream affordance +and this is not a place for a one-off, and that the short-term path agreed with +Max was a client-side Redis connection. That is an ownership question, not a +technical one, and it is not mine to settle here. + +## What the prototype assumes in the meantime + +An external shared store, with `stream_log` as the working implementation. Not +because it should ship, but because the benchmark needs a substrate that does +not fall over inside the workload. When CHASM event storage lands, this is the +piece that gets deleted. + +The consequence already recorded: the three persistence methods sit on +`ExecutionStore` (`persistence_interface.go:168-175`), which presumes the log is +shard-local. Under any external store that is the wrong home. Given the answer +above, moving it is probably wasted work, so it stays as it is. + +## What replication does, until then + +Nothing. A standby holds a frontier and no bytes, so a failover breaks every +reader. That is now a documented limitation of a prototype rather than a gap to +close, because the replication of stream bytes arrives with CHASM event storage +in H2. + +The protocol sketch that used to be the point of this file is kept below, +because it is cheap to keep and it is a concrete answer to one question that +CHASM event storage will have to answer too: how a receiver tells a sender which +byte ranges it already holds. + +Three properties, all from keying a record by the offset it starts at. Applying +a record is idempotent, so duplicates and retries are free. Records are +independent, so out-of-order arrival is harmless. A record is self-describing, +so a receiver needs no context to place it. Together they make shipping the +payload viable rather than fetching it back the way history events are. + +Carry the records in the message that already carries the frontier, and apply +records before state. Ordering is then free, which matters because it is the +thing most likely to be got wrong: shipped separately, the frontier can arrive +first and the standby holds offsets whose bytes never landed. + +For which records to ship, have the receiver report the offset it holds through, +per collection, and ship from there to the frontier. It needs a back-channel +that does not exist, since receiver progress travels today as a per-shard +task-id acknowledgement. It pays for itself by making the byte cap free: a +message that cannot carry the whole range carries a prefix, and the receiver's +next report resumes exactly there, with no continuation token and no resumption +state. diff --git a/chasm/lib/stream/config.go b/chasm/lib/stream/config.go new file mode 100644 index 00000000000..7168d46b92d --- /dev/null +++ b/chasm/lib/stream/config.go @@ -0,0 +1,69 @@ +package stream + +import "time" + +// DefaultMaxMessagesPerPoll bounds a read page when the caller does not. +const DefaultMaxMessagesPerPoll = 1000 + +// MaxMessagesPerBatch bounds one append. It is not only an admission limit: a +// node ID is the first offset of its batch, so to serve a read starting inside +// a batch the reader has to find the node that contains it. Bounding the batch +// bounds how far back it has to start, which turns an unbounded scan into a +// fixed overread. +const MaxMessagesPerBatch = 1000 + +// LongPollTimeout matches the convention used by the history long polls: on +// expiry the caller gets an empty response and polls again, rather than an +// error it would have to special-case. +const LongPollTimeout = 20 * time.Second + +// LongPollBuffer leaves room to return an empty response before the caller's +// own deadline fires. +const LongPollBuffer = 3 * time.Second + +// MaxConsumeItemsPerTask bounds one Workflow Task's slice. A byte cap alone is +// not enough: a burst of tiny messages stays under it while still making one +// task's drain arbitrarily long. Whichever bound binds first, the rest is +// delivered on the following task. +const MaxConsumeItemsPerTask = 1000 + +// MaxConsumeBytesPerTask bounds one Workflow Task's slice by size. Paired with +// MaxConsumeItemsPerTask because neither bound alone is enough: a burst of tiny +// messages slips under the byte budget, and a few large ones slip under the +// item count. +const MaxConsumeBytesPerTask = 2 << 20 + +// MaxProducersPerStream bounds the per-producer dedup table. The table is part +// of the component state written on every append, so a caller that sends a +// fresh producer id per request would grow the state until the mutable-state +// size limit rejects every further append, leaving the stream unwritable for +// good. The bound turns that into a clear error on the offending call. +const MaxProducersPerStream = 1000 + +// MaxConsumersPerStream bounds the registered consumer table for the same +// reason. Each consumer also holds a truncation floor, so an unbounded table +// would pin storage as well as grow state. +const MaxConsumersPerStream = 1000 + +// MaxListPageSize bounds a visibility page when the caller does not. +const MaxListPageSize = 1000 + +// MaxMessageBytes bounds one message. A message is never split, so this is also +// the smallest unit a reader can be asked to materialise. +const MaxMessageBytes = 1 << 20 + +// MaxBatchBytes bounds one append. It is deliberately equal to +// MaxConsumeBytesPerTask: a batch is written as one node and read back whole, +// so a batch larger than a task's byte budget could never be delivered. +const MaxBatchBytes = MaxConsumeBytesPerTask + +// MaxOwnedStreamsPerWorkflow bounds how many named streams one execution can +// carry. Each is a component in the workflow's mutable state, so an unbounded +// count grows that state until the size limit terminates the execution. The +// name comes from the caller, and any caller in the namespace can pick a new +// one, which is what makes this reachable from outside. +const MaxOwnedStreamsPerWorkflow = 100 + +// MaxStreamNameLength bounds a name before it becomes a map key in mutable +// state, for the same reason. +const MaxStreamNameLength = 255 diff --git a/chasm/lib/stream/cursor.go b/chasm/lib/stream/cursor.go new file mode 100644 index 00000000000..265ec33daae --- /dev/null +++ b/chasm/lib/stream/cursor.go @@ -0,0 +1,167 @@ +package stream + +import ( + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" +) + +// Cursor is a consuming workflow's position in a stream. +// +// It is a subcomponent of the consumer, not of the stream. That placement is +// the whole point: the consumer's mutable state and its History events commit +// in one transaction, so folding a delivered range into the cursor lands +// atomically with the event that records the range. Holding the cursor on the +// stream instead would make every advance a cross-execution write, and a crash +// between the two writes would either redeliver a range or skip it silently. +type Cursor struct { + chasm.UnimplementedComponent + + State *streampb.WorkflowStreamCursor +} + +type NewCursorRequest struct { + StreamID string + // External marks a stream in another execution, whose frontier this + // workflow is told about rather than reads. + External bool + CollectionID string + BucketSize int64 + + // Where to start reading. Resolving "from the tail" against the stream's + // head happens before this is called, so the value recorded here is already + // a fact rather than a reading that would differ on replay. + StartOffset int64 +} + +func NewCursor(_ chasm.MutableContext, req NewCursorRequest) (*Cursor, error) { + if req.StreamID == "" { + return nil, serviceerror.NewInvalidArgument("stream id is required") + } + if req.CollectionID == "" { + return nil, serviceerror.NewInvalidArgument("collection id is required") + } + if req.BucketSize <= 0 { + return nil, serviceerror.NewInvalidArgument("bucket size must be positive") + } + if req.StartOffset < 0 { + return nil, serviceerror.NewInvalidArgument("start offset cannot be negative") + } + + return &Cursor{ + State: &streampb.WorkflowStreamCursor{ + StreamId: req.StreamID, + CollectionId: req.CollectionID, + BucketSize: req.BucketSize, + Offset: req.StartOffset, + StartOffset: req.StartOffset, + External: req.External, + KnownHead: req.StartOffset, + }, + }, nil +} + +// LifecycleState reports the cursor as running for as long as the workflow +// holding it exists. Deregistration is an explicit act, not a state the +// component reaches on its own. +func (c *Cursor) LifecycleState(_ chasm.Context) chasm.LifecycleState { + return chasm.LifecycleStateRunning +} + +// Offset is the next offset that has not yet been delivered and folded in. +func (c *Cursor) Offset() int64 { + return c.State.Offset +} + +func (c *Cursor) StreamID() string { + return c.State.StreamId +} + +func (c *Cursor) CollectionID() string { + return c.State.CollectionId +} + +func (c *Cursor) BucketSize() int64 { + return c.State.BucketSize +} + +// StagePending records the range attached to the workflow task now in flight. +// +// A redelivery overwrites whatever was staged before. That is safe because a +// range only becomes history when the task completes: if the previous task +// failed or timed out, nothing was recorded, so the replacement range is the +// first one the workflow will ever have observed at this point. +func (c *Cursor) StagePending(_ chasm.MutableContext, from int64, to int64) error { + if from < c.State.Offset { + return serviceerror.NewInvalidArgumentf( + "cannot deliver from offset %d, cursor is already at %d", from, c.State.Offset) + } + if to < from { + return serviceerror.NewInvalidArgumentf("range end %d precedes range start %d", to, from) + } + + c.State.PendingFrom = from + c.State.PendingTo = to + c.State.HasPending = true + return nil +} + +// Pending reports the staged range. The second result distinguishes "no task in +// flight" from "a task in flight that was given nothing", which are different +// facts: the latter must still be recorded. +func (c *Cursor) Pending() (from int64, to int64, ok bool) { + if !c.State.HasPending { + return 0, 0, false + } + return c.State.PendingFrom, c.State.PendingTo, true +} + +// Commit folds the staged range into the cursor and returns it for recording. +// Caller writes the returned range onto the event that closes the task, in the +// same transaction that persists this advance. +func (c *Cursor) Commit(_ chasm.MutableContext) (from int64, to int64, ok bool) { + if !c.State.HasPending { + return 0, 0, false + } + + from, to = c.State.PendingFrom, c.State.PendingTo + c.State.Offset = to + c.State.PendingFrom = 0 + c.State.PendingTo = 0 + c.State.HasPending = false + return from, to, true +} + +// Abandon drops a staged range without advancing, for a task that will never +// complete. The next delivery re-reads from the unchanged cursor. +func (c *Cursor) Abandon(_ chasm.MutableContext) { + c.State.PendingFrom = 0 + c.State.PendingTo = 0 + c.State.HasPending = false +} + +// IsExternal reports whether the stream lives in another execution. +func (c *Cursor) IsExternal() bool { + return c.State.External +} + +// KnownHead is the stream's frontier as last pushed to this workflow. +func (c *Cursor) KnownHead() int64 { + return c.State.KnownHead +} + +// AdvanceKnownHead moves the recorded frontier forward. It never moves back: a +// stale push arriving after a fresher one must not hide offsets already known +// to exist. +func (c *Cursor) AdvanceKnownHead(_ chasm.MutableContext, head int64) { + if head > c.State.KnownHead { + c.State.KnownHead = head + } +} + +// StartOffset is where this subscription began reading. Replay needs it to tell +// a consumer that has committed nothing apart from one whose recording events +// are simply not in the history page it was handed. +func (c *Cursor) StartOffset() int64 { + return c.State.StartOffset +} diff --git a/chasm/lib/stream/cursor_test.go b/chasm/lib/stream/cursor_test.go new file mode 100644 index 00000000000..a5ca5314500 --- /dev/null +++ b/chasm/lib/stream/cursor_test.go @@ -0,0 +1,118 @@ +package stream + +import ( + "testing" + + "github.com/stretchr/testify/require" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" +) + +func newTestCursor(offset int64) *Cursor { + return &Cursor{ + State: &streampb.WorkflowStreamCursor{ + StreamId: "s-1", + CollectionId: "col-1", + BucketSize: DefaultBucketSize, + Offset: offset, + }, + } +} + +func TestNewCursorRejectsIncompleteRequests(t *testing.T) { + cases := map[string]NewCursorRequest{ + "no stream id": {CollectionID: "col-1", BucketSize: 10}, + "no collection id": {StreamID: "s-1", BucketSize: 10}, + "zero bucket size": {StreamID: "s-1", CollectionID: "col-1"}, + "negative start": {StreamID: "s-1", CollectionID: "col-1", BucketSize: 10, StartOffset: -1}, + } + + for name, req := range cases { + t.Run(name, func(t *testing.T) { + _, err := NewCursor(nil, req) + require.Error(t, err) + }) + } +} + +func TestCursorCommitAdvancesAndClears(t *testing.T) { + c := newTestCursor(4) + + require.NoError(t, c.StagePending(nil, 4, 7)) + + from, to, ok := c.Pending() + require.True(t, ok) + require.Equal(t, int64(4), from) + require.Equal(t, int64(7), to) + + from, to, ok = c.Commit(nil) + require.True(t, ok) + require.Equal(t, int64(4), from) + require.Equal(t, int64(7), to) + require.Equal(t, int64(7), c.Offset()) + + _, _, ok = c.Pending() + require.False(t, ok, "a committed range must not be staged twice") + + _, _, ok = c.Commit(nil) + require.False(t, ok, "committing again must not re-record the range") +} + +// The distinction this pins is the one §8.2 of the design turns on: a task that +// observed nothing still has to be recorded, so an empty range is a pending +// range, not the absence of one. +func TestCursorTreatsAnEmptyRangeAsAFact(t *testing.T) { + c := newTestCursor(9) + + _, _, ok := c.Pending() + require.False(t, ok, "no task in flight yet") + + require.NoError(t, c.StagePending(nil, 9, 9)) + + from, to, ok := c.Pending() + require.True(t, ok, "a task given nothing is still a task that must be recorded") + require.Equal(t, from, to) + + from, to, ok = c.Commit(nil) + require.True(t, ok) + require.Equal(t, int64(9), from) + require.Equal(t, int64(9), to) + require.Equal(t, int64(9), c.Offset(), "an empty range must not move the cursor") +} + +func TestCursorRejectsARangeBehindItself(t *testing.T) { + c := newTestCursor(12) + + err := c.StagePending(nil, 11, 14) + require.ErrorContains(t, err, "cursor is already at 12") + + err = c.StagePending(nil, 12, 11) + require.ErrorContains(t, err, "precedes range start") +} + +// A task that failed recorded nothing, so the range it was given never became +// history and the replacement is free to differ. +func TestCursorRedeliveryReplacesTheStagedRange(t *testing.T) { + c := newTestCursor(2) + + require.NoError(t, c.StagePending(nil, 2, 5)) + require.NoError(t, c.StagePending(nil, 2, 9)) + + from, to, ok := c.Pending() + require.True(t, ok) + require.Equal(t, int64(2), from) + require.Equal(t, int64(9), to) + require.Equal(t, int64(2), c.Offset(), "staging alone must never advance the cursor") +} + +func TestCursorAbandonLeavesTheOffsetAlone(t *testing.T) { + c := newTestCursor(3) + + require.NoError(t, c.StagePending(nil, 3, 8)) + c.Abandon(nil) + + _, _, ok := c.Pending() + require.False(t, ok) + require.Equal(t, int64(3), c.Offset()) + + require.NoError(t, c.StagePending(nil, 3, 6), "the next delivery re-reads from the unchanged cursor") +} diff --git a/chasm/lib/stream/gen/streampb/v1/message.go-helpers.pb.go b/chasm/lib/stream/gen/streampb/v1/message.go-helpers.pb.go new file mode 100644 index 00000000000..28778a8fce9 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/message.go-helpers.pb.go @@ -0,0 +1,101 @@ +// Code generated by protoc-gen-go-helpers. DO NOT EDIT. +package streampb + +import ( + "fmt" + + "google.golang.org/protobuf/proto" +) + +// Marshal an object of type StreamMessage to the protobuf v3 wire format +func (val *StreamMessage) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StreamMessage from the protobuf v3 wire format +func (val *StreamMessage) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StreamMessage) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StreamMessage values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StreamMessage) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StreamMessage + switch t := that.(type) { + case *StreamMessage: + that1 = t + case StreamMessage: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type StreamMessageBatch to the protobuf v3 wire format +func (val *StreamMessageBatch) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StreamMessageBatch from the protobuf v3 wire format +func (val *StreamMessageBatch) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StreamMessageBatch) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StreamMessageBatch values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StreamMessageBatch) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StreamMessageBatch + switch t := that.(type) { + case *StreamMessageBatch: + that1 = t + case StreamMessageBatch: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +var ( + StreamMessageKind_shorthandValue = map[string]int32{ + "Unspecified": 0, + "Data": 1, + "Flush": 2, + } +) + +// StreamMessageKindFromString parses a StreamMessageKind value from either the protojson +// canonical SCREAMING_CASE enum or the traditional temporal PascalCase enum to StreamMessageKind +func StreamMessageKindFromString(s string) (StreamMessageKind, error) { + if v, ok := StreamMessageKind_value[s]; ok { + return StreamMessageKind(v), nil + } else if v, ok := StreamMessageKind_shorthandValue[s]; ok { + return StreamMessageKind(v), nil + } + return StreamMessageKind(0), fmt.Errorf("%s is not a valid StreamMessageKind", s) +} diff --git a/chasm/lib/stream/gen/streampb/v1/message.pb.go b/chasm/lib/stream/gen/streampb/v1/message.pb.go new file mode 100644 index 00000000000..6ab9fdab0a0 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/message.pb.go @@ -0,0 +1,305 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/stream/proto/v1/message.proto + +package streampb + +import ( + reflect "reflect" + "strconv" + sync "sync" + unsafe "unsafe" + + v1 "go.temporal.io/api/common/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type StreamMessageKind int32 + +const ( + STREAM_MESSAGE_KIND_UNSPECIFIED StreamMessageKind = 0 + STREAM_MESSAGE_KIND_DATA StreamMessageKind = 1 + // Producer signalling a delivery boundary. Carries no body and consumes an + // offset like any other message. A consumer uses it to end a turn without + // waiting out an idle timeout. + STREAM_MESSAGE_KIND_FLUSH StreamMessageKind = 2 +) + +// Enum value maps for StreamMessageKind. +var ( + StreamMessageKind_name = map[int32]string{ + 0: "STREAM_MESSAGE_KIND_UNSPECIFIED", + 1: "STREAM_MESSAGE_KIND_DATA", + 2: "STREAM_MESSAGE_KIND_FLUSH", + } + StreamMessageKind_value = map[string]int32{ + "STREAM_MESSAGE_KIND_UNSPECIFIED": 0, + "STREAM_MESSAGE_KIND_DATA": 1, + "STREAM_MESSAGE_KIND_FLUSH": 2, + } +) + +func (x StreamMessageKind) Enum() *StreamMessageKind { + p := new(StreamMessageKind) + *p = x + return p +} + +func (x StreamMessageKind) String() string { + switch x { + case STREAM_MESSAGE_KIND_UNSPECIFIED: + return "Unspecified" + case STREAM_MESSAGE_KIND_DATA: + return "Data" + case STREAM_MESSAGE_KIND_FLUSH: + return "Flush" + default: + return strconv.Itoa(int(x)) + } + +} + +func (StreamMessageKind) Descriptor() protoreflect.EnumDescriptor { + return file_temporal_server_chasm_lib_stream_proto_v1_message_proto_enumTypes[0].Descriptor() +} + +func (StreamMessageKind) Type() protoreflect.EnumType { + return &file_temporal_server_chasm_lib_stream_proto_v1_message_proto_enumTypes[0] +} + +func (x StreamMessageKind) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use StreamMessageKind.Descriptor instead. +func (StreamMessageKind) EnumDescriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_message_proto_rawDescGZIP(), []int{0} +} + +type StreamMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Body *v1.Payload `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + // Producer-supplied provenance. The server does not populate this today. + Metadata map[string]*v1.Payload `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Topic string `protobuf:"bytes,3,opt,name=topic,proto3" json:"topic,omitempty"` + // Position within this topic. The global offset orders the whole stream; + // this lets a consumer reason about one topic without decoding the rest. + TopicSequence int64 `protobuf:"varint,4,opt,name=topic_sequence,json=topicSequence,proto3" json:"topic_sequence,omitempty"` + Kind StreamMessageKind `protobuf:"varint,5,opt,name=kind,proto3,enum=temporal.server.chasm.lib.stream.proto.v1.StreamMessageKind" json:"kind,omitempty"` + // Position in the whole stream, set on read and never stored. A consumer + // that resumes at message granularity needs it, and a topic-filtered read + // leaves gaps that make it underivable from the response alone. + Offset int64 `protobuf:"varint,6,opt,name=offset,proto3" json:"offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamMessage) Reset() { + *x = StreamMessage{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_message_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamMessage) ProtoMessage() {} + +func (x *StreamMessage) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_message_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamMessage.ProtoReflect.Descriptor instead. +func (*StreamMessage) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_message_proto_rawDescGZIP(), []int{0} +} + +func (x *StreamMessage) GetBody() *v1.Payload { + if x != nil { + return x.Body + } + return nil +} + +func (x *StreamMessage) GetMetadata() map[string]*v1.Payload { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *StreamMessage) GetTopic() string { + if x != nil { + return x.Topic + } + return "" +} + +func (x *StreamMessage) GetTopicSequence() int64 { + if x != nil { + return x.TopicSequence + } + return 0 +} + +func (x *StreamMessage) GetKind() StreamMessageKind { + if x != nil { + return x.Kind + } + return STREAM_MESSAGE_KIND_UNSPECIFIED +} + +func (x *StreamMessage) GetOffset() int64 { + if x != nil { + return x.Offset + } + return 0 +} + +// One append is one batch, and one batch is one log node. The server stores +// this serialized and opaque; it decodes only to trim a partial first page or +// to apply a topic filter. +type StreamMessageBatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Messages []*StreamMessage `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamMessageBatch) Reset() { + *x = StreamMessageBatch{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_message_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamMessageBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamMessageBatch) ProtoMessage() {} + +func (x *StreamMessageBatch) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_message_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamMessageBatch.ProtoReflect.Descriptor instead. +func (*StreamMessageBatch) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_message_proto_rawDescGZIP(), []int{1} +} + +func (x *StreamMessageBatch) GetMessages() []*StreamMessage { + if x != nil { + return x.Messages + } + return nil +} + +var File_temporal_server_chasm_lib_stream_proto_v1_message_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_stream_proto_v1_message_proto_rawDesc = "" + + "\n" + + "7temporal/server/chasm/lib/stream/proto/v1/message.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a$temporal/api/common/v1/message.proto\"\xad\x03\n" + + "\rStreamMessage\x123\n" + + "\x04body\x18\x01 \x01(\v2\x1f.temporal.api.common.v1.PayloadR\x04body\x12b\n" + + "\bmetadata\x18\x02 \x03(\v2F.temporal.server.chasm.lib.stream.proto.v1.StreamMessage.MetadataEntryR\bmetadata\x12\x14\n" + + "\x05topic\x18\x03 \x01(\tR\x05topic\x12%\n" + + "\x0etopic_sequence\x18\x04 \x01(\x03R\rtopicSequence\x12P\n" + + "\x04kind\x18\x05 \x01(\x0e2<.temporal.server.chasm.lib.stream.proto.v1.StreamMessageKindR\x04kind\x12\x16\n" + + "\x06offset\x18\x06 \x01(\x03R\x06offset\x1a\\\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x125\n" + + "\x05value\x18\x02 \x01(\v2\x1f.temporal.api.common.v1.PayloadR\x05value:\x028\x01\"j\n" + + "\x12StreamMessageBatch\x12T\n" + + "\bmessages\x18\x01 \x03(\v28.temporal.server.chasm.lib.stream.proto.v1.StreamMessageR\bmessages*u\n" + + "\x11StreamMessageKind\x12#\n" + + "\x1fSTREAM_MESSAGE_KIND_UNSPECIFIED\x10\x00\x12\x1c\n" + + "\x18STREAM_MESSAGE_KIND_DATA\x10\x01\x12\x1d\n" + + "\x19STREAM_MESSAGE_KIND_FLUSH\x10\x02B>Z temporal.api.common.v1.Payload + 3, // 1: temporal.server.chasm.lib.stream.proto.v1.StreamMessage.metadata:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamMessage.MetadataEntry + 0, // 2: temporal.server.chasm.lib.stream.proto.v1.StreamMessage.kind:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamMessageKind + 1, // 3: temporal.server.chasm.lib.stream.proto.v1.StreamMessageBatch.messages:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamMessage + 4, // 4: temporal.server.chasm.lib.stream.proto.v1.StreamMessage.MetadataEntry.value:type_name -> temporal.api.common.v1.Payload + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_temporal_server_chasm_lib_stream_proto_v1_message_proto_init() } +func file_temporal_server_chasm_lib_stream_proto_v1_message_proto_init() { + if File_temporal_server_chasm_lib_stream_proto_v1_message_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_stream_proto_v1_message_proto_rawDesc), len(file_temporal_server_chasm_lib_stream_proto_v1_message_proto_rawDesc)), + NumEnums: 1, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_temporal_server_chasm_lib_stream_proto_v1_message_proto_goTypes, + DependencyIndexes: file_temporal_server_chasm_lib_stream_proto_v1_message_proto_depIdxs, + EnumInfos: file_temporal_server_chasm_lib_stream_proto_v1_message_proto_enumTypes, + MessageInfos: file_temporal_server_chasm_lib_stream_proto_v1_message_proto_msgTypes, + }.Build() + File_temporal_server_chasm_lib_stream_proto_v1_message_proto = out.File + file_temporal_server_chasm_lib_stream_proto_v1_message_proto_goTypes = nil + file_temporal_server_chasm_lib_stream_proto_v1_message_proto_depIdxs = nil +} diff --git a/chasm/lib/stream/gen/streampb/v1/namespace.go b/chasm/lib/stream/gen/streampb/v1/namespace.go new file mode 100644 index 00000000000..dd465ef065e --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/namespace.go @@ -0,0 +1,75 @@ +package streampb + +// Every RPC here carries its namespace inside `frontend_request` rather than at +// the top level, because the top-level field is the resolved namespace id the +// frontend fills in before routing. +// +// The server's interceptors find a request's namespace by asserting it to +// `interceptor.NamespaceNameGetter`, which wants `GetNamespace() string` on the +// request itself. Without these the assertion falls through to the id getter, +// which at the frontend is still empty, so namespace rate limits, request +// validation, the authorization target, redirection and the long-poll deadline +// all resolve to the empty namespace and silently do nothing. +// +// The generator has no way to express "read it from this nested field", so the +// methods are written here, next to the generated types they belong to. + +func (x *CreateStreamRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *AddMessagesRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *FinishWritingRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *SubscribeWorkflowRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *PollMessagesRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *DescribeStreamRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *PollWorkflowMessagesRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *DescribeWorkflowStreamRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *AddWorkflowMessagesRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *RegisterStreamConsumerRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *AdvanceConsumerHeadRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *CloseStreamRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *TruncateStreamRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *ListStreamsRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} + +func (x *DeleteStreamRequest) GetNamespace() string { + return x.GetFrontendRequest().GetNamespace() +} diff --git a/chasm/lib/stream/gen/streampb/v1/namespace_test.go b/chasm/lib/stream/gen/streampb/v1/namespace_test.go new file mode 100644 index 00000000000..b88bcaf856c --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/namespace_test.go @@ -0,0 +1,38 @@ +package streampb + +import ( + "testing" + + "go.temporal.io/server/common/rpc/interceptor" + "google.golang.org/protobuf/reflect/protoregistry" +) + +// Every request carrying a frontend_request must expose its namespace to the +// interceptors. Driven off the descriptor rather than a hand-written list so a +// new RPC fails here instead of silently opting out of namespace rate limits, +// validation, authorization and redirection. +func TestEveryRoutedRequestExposesNamespace(t *testing.T) { + fd := File_temporal_server_chasm_lib_stream_proto_v1_request_response_proto + messages := fd.Messages() + + checked := 0 + for i := 0; i < messages.Len(); i++ { + md := messages.Get(i) + if md.Fields().ByName("frontend_request") == nil { + continue + } + mt, err := protoregistry.GlobalTypes.FindMessageByName(md.FullName()) + if err != nil { + t.Fatalf("%s is not registered: %v", md.FullName(), err) + } + msg := mt.New().Interface() + if _, ok := msg.(interceptor.NamespaceNameGetter); !ok { + t.Errorf("%s has a frontend_request but no GetNamespace; add it in namespace.go", md.FullName()) + } + checked++ + } + + if checked == 0 { + t.Fatal("found no routed requests, so this test is not checking anything") + } +} diff --git a/chasm/lib/stream/gen/streampb/v1/request_response.go-helpers.pb.go b/chasm/lib/stream/gen/streampb/v1/request_response.go-helpers.pb.go new file mode 100644 index 00000000000..7c11d9434cd --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/request_response.go-helpers.pb.go @@ -0,0 +1,2152 @@ +// Code generated by protoc-gen-go-helpers. DO NOT EDIT. +package streampb + +import ( + "google.golang.org/protobuf/proto" +) + +// Marshal an object of type CreateStreamInput to the protobuf v3 wire format +func (val *CreateStreamInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CreateStreamInput from the protobuf v3 wire format +func (val *CreateStreamInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CreateStreamInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CreateStreamInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CreateStreamInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CreateStreamInput + switch t := that.(type) { + case *CreateStreamInput: + that1 = t + case CreateStreamInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type CreateStreamOutput to the protobuf v3 wire format +func (val *CreateStreamOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CreateStreamOutput from the protobuf v3 wire format +func (val *CreateStreamOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CreateStreamOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CreateStreamOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CreateStreamOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CreateStreamOutput + switch t := that.(type) { + case *CreateStreamOutput: + that1 = t + case CreateStreamOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AddMessagesInput to the protobuf v3 wire format +func (val *AddMessagesInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AddMessagesInput from the protobuf v3 wire format +func (val *AddMessagesInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AddMessagesInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AddMessagesInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AddMessagesInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AddMessagesInput + switch t := that.(type) { + case *AddMessagesInput: + that1 = t + case AddMessagesInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AddMessagesOutput to the protobuf v3 wire format +func (val *AddMessagesOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AddMessagesOutput from the protobuf v3 wire format +func (val *AddMessagesOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AddMessagesOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AddMessagesOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AddMessagesOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AddMessagesOutput + switch t := that.(type) { + case *AddMessagesOutput: + that1 = t + case AddMessagesOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type FinishWritingInput to the protobuf v3 wire format +func (val *FinishWritingInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type FinishWritingInput from the protobuf v3 wire format +func (val *FinishWritingInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *FinishWritingInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two FinishWritingInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *FinishWritingInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *FinishWritingInput + switch t := that.(type) { + case *FinishWritingInput: + that1 = t + case FinishWritingInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type FinishWritingOutput to the protobuf v3 wire format +func (val *FinishWritingOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type FinishWritingOutput from the protobuf v3 wire format +func (val *FinishWritingOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *FinishWritingOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two FinishWritingOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *FinishWritingOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *FinishWritingOutput + switch t := that.(type) { + case *FinishWritingOutput: + that1 = t + case FinishWritingOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type SubscribeWorkflowInput to the protobuf v3 wire format +func (val *SubscribeWorkflowInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type SubscribeWorkflowInput from the protobuf v3 wire format +func (val *SubscribeWorkflowInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *SubscribeWorkflowInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two SubscribeWorkflowInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *SubscribeWorkflowInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *SubscribeWorkflowInput + switch t := that.(type) { + case *SubscribeWorkflowInput: + that1 = t + case SubscribeWorkflowInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type SubscribeWorkflowOutput to the protobuf v3 wire format +func (val *SubscribeWorkflowOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type SubscribeWorkflowOutput from the protobuf v3 wire format +func (val *SubscribeWorkflowOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *SubscribeWorkflowOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two SubscribeWorkflowOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *SubscribeWorkflowOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *SubscribeWorkflowOutput + switch t := that.(type) { + case *SubscribeWorkflowOutput: + that1 = t + case SubscribeWorkflowOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollMessagesInput to the protobuf v3 wire format +func (val *PollMessagesInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollMessagesInput from the protobuf v3 wire format +func (val *PollMessagesInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollMessagesInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollMessagesInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollMessagesInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollMessagesInput + switch t := that.(type) { + case *PollMessagesInput: + that1 = t + case PollMessagesInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollMessagesOutput to the protobuf v3 wire format +func (val *PollMessagesOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollMessagesOutput from the protobuf v3 wire format +func (val *PollMessagesOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollMessagesOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollMessagesOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollMessagesOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollMessagesOutput + switch t := that.(type) { + case *PollMessagesOutput: + that1 = t + case PollMessagesOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeStreamInput to the protobuf v3 wire format +func (val *DescribeStreamInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeStreamInput from the protobuf v3 wire format +func (val *DescribeStreamInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeStreamInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeStreamInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeStreamInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeStreamInput + switch t := that.(type) { + case *DescribeStreamInput: + that1 = t + case DescribeStreamInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollWorkflowMessagesInput to the protobuf v3 wire format +func (val *PollWorkflowMessagesInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollWorkflowMessagesInput from the protobuf v3 wire format +func (val *PollWorkflowMessagesInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollWorkflowMessagesInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollWorkflowMessagesInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollWorkflowMessagesInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollWorkflowMessagesInput + switch t := that.(type) { + case *PollWorkflowMessagesInput: + that1 = t + case PollWorkflowMessagesInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeWorkflowStreamInput to the protobuf v3 wire format +func (val *DescribeWorkflowStreamInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeWorkflowStreamInput from the protobuf v3 wire format +func (val *DescribeWorkflowStreamInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeWorkflowStreamInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeWorkflowStreamInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeWorkflowStreamInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeWorkflowStreamInput + switch t := that.(type) { + case *DescribeWorkflowStreamInput: + that1 = t + case DescribeWorkflowStreamInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AddWorkflowMessagesInput to the protobuf v3 wire format +func (val *AddWorkflowMessagesInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AddWorkflowMessagesInput from the protobuf v3 wire format +func (val *AddWorkflowMessagesInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AddWorkflowMessagesInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AddWorkflowMessagesInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AddWorkflowMessagesInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AddWorkflowMessagesInput + switch t := that.(type) { + case *AddWorkflowMessagesInput: + that1 = t + case AddWorkflowMessagesInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeStreamOutput to the protobuf v3 wire format +func (val *DescribeStreamOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeStreamOutput from the protobuf v3 wire format +func (val *DescribeStreamOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeStreamOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeStreamOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeStreamOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeStreamOutput + switch t := that.(type) { + case *DescribeStreamOutput: + that1 = t + case DescribeStreamOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type CloseStreamInput to the protobuf v3 wire format +func (val *CloseStreamInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CloseStreamInput from the protobuf v3 wire format +func (val *CloseStreamInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CloseStreamInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CloseStreamInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CloseStreamInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CloseStreamInput + switch t := that.(type) { + case *CloseStreamInput: + that1 = t + case CloseStreamInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type CloseStreamOutput to the protobuf v3 wire format +func (val *CloseStreamOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CloseStreamOutput from the protobuf v3 wire format +func (val *CloseStreamOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CloseStreamOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CloseStreamOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CloseStreamOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CloseStreamOutput + switch t := that.(type) { + case *CloseStreamOutput: + that1 = t + case CloseStreamOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type TruncateStreamInput to the protobuf v3 wire format +func (val *TruncateStreamInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type TruncateStreamInput from the protobuf v3 wire format +func (val *TruncateStreamInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *TruncateStreamInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two TruncateStreamInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *TruncateStreamInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *TruncateStreamInput + switch t := that.(type) { + case *TruncateStreamInput: + that1 = t + case TruncateStreamInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type TruncateStreamOutput to the protobuf v3 wire format +func (val *TruncateStreamOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type TruncateStreamOutput from the protobuf v3 wire format +func (val *TruncateStreamOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *TruncateStreamOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two TruncateStreamOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *TruncateStreamOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *TruncateStreamOutput + switch t := that.(type) { + case *TruncateStreamOutput: + that1 = t + case TruncateStreamOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DeleteStreamInput to the protobuf v3 wire format +func (val *DeleteStreamInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DeleteStreamInput from the protobuf v3 wire format +func (val *DeleteStreamInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DeleteStreamInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DeleteStreamInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DeleteStreamInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DeleteStreamInput + switch t := that.(type) { + case *DeleteStreamInput: + that1 = t + case DeleteStreamInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DeleteStreamOutput to the protobuf v3 wire format +func (val *DeleteStreamOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DeleteStreamOutput from the protobuf v3 wire format +func (val *DeleteStreamOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DeleteStreamOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DeleteStreamOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DeleteStreamOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DeleteStreamOutput + switch t := that.(type) { + case *DeleteStreamOutput: + that1 = t + case DeleteStreamOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type CreateStreamRequest to the protobuf v3 wire format +func (val *CreateStreamRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CreateStreamRequest from the protobuf v3 wire format +func (val *CreateStreamRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CreateStreamRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CreateStreamRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CreateStreamRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CreateStreamRequest + switch t := that.(type) { + case *CreateStreamRequest: + that1 = t + case CreateStreamRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type CreateStreamResponse to the protobuf v3 wire format +func (val *CreateStreamResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CreateStreamResponse from the protobuf v3 wire format +func (val *CreateStreamResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CreateStreamResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CreateStreamResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CreateStreamResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CreateStreamResponse + switch t := that.(type) { + case *CreateStreamResponse: + that1 = t + case CreateStreamResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AddMessagesRequest to the protobuf v3 wire format +func (val *AddMessagesRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AddMessagesRequest from the protobuf v3 wire format +func (val *AddMessagesRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AddMessagesRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AddMessagesRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AddMessagesRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AddMessagesRequest + switch t := that.(type) { + case *AddMessagesRequest: + that1 = t + case AddMessagesRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AddMessagesResponse to the protobuf v3 wire format +func (val *AddMessagesResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AddMessagesResponse from the protobuf v3 wire format +func (val *AddMessagesResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AddMessagesResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AddMessagesResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AddMessagesResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AddMessagesResponse + switch t := that.(type) { + case *AddMessagesResponse: + that1 = t + case AddMessagesResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type FinishWritingRequest to the protobuf v3 wire format +func (val *FinishWritingRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type FinishWritingRequest from the protobuf v3 wire format +func (val *FinishWritingRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *FinishWritingRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two FinishWritingRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *FinishWritingRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *FinishWritingRequest + switch t := that.(type) { + case *FinishWritingRequest: + that1 = t + case FinishWritingRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type FinishWritingResponse to the protobuf v3 wire format +func (val *FinishWritingResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type FinishWritingResponse from the protobuf v3 wire format +func (val *FinishWritingResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *FinishWritingResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two FinishWritingResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *FinishWritingResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *FinishWritingResponse + switch t := that.(type) { + case *FinishWritingResponse: + that1 = t + case FinishWritingResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type SubscribeWorkflowRequest to the protobuf v3 wire format +func (val *SubscribeWorkflowRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type SubscribeWorkflowRequest from the protobuf v3 wire format +func (val *SubscribeWorkflowRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *SubscribeWorkflowRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two SubscribeWorkflowRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *SubscribeWorkflowRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *SubscribeWorkflowRequest + switch t := that.(type) { + case *SubscribeWorkflowRequest: + that1 = t + case SubscribeWorkflowRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type SubscribeWorkflowResponse to the protobuf v3 wire format +func (val *SubscribeWorkflowResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type SubscribeWorkflowResponse from the protobuf v3 wire format +func (val *SubscribeWorkflowResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *SubscribeWorkflowResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two SubscribeWorkflowResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *SubscribeWorkflowResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *SubscribeWorkflowResponse + switch t := that.(type) { + case *SubscribeWorkflowResponse: + that1 = t + case SubscribeWorkflowResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollMessagesRequest to the protobuf v3 wire format +func (val *PollMessagesRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollMessagesRequest from the protobuf v3 wire format +func (val *PollMessagesRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollMessagesRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollMessagesRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollMessagesRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollMessagesRequest + switch t := that.(type) { + case *PollMessagesRequest: + that1 = t + case PollMessagesRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollMessagesResponse to the protobuf v3 wire format +func (val *PollMessagesResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollMessagesResponse from the protobuf v3 wire format +func (val *PollMessagesResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollMessagesResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollMessagesResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollMessagesResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollMessagesResponse + switch t := that.(type) { + case *PollMessagesResponse: + that1 = t + case PollMessagesResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeStreamRequest to the protobuf v3 wire format +func (val *DescribeStreamRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeStreamRequest from the protobuf v3 wire format +func (val *DescribeStreamRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeStreamRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeStreamRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeStreamRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeStreamRequest + switch t := that.(type) { + case *DescribeStreamRequest: + that1 = t + case DescribeStreamRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeStreamResponse to the protobuf v3 wire format +func (val *DescribeStreamResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeStreamResponse from the protobuf v3 wire format +func (val *DescribeStreamResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeStreamResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeStreamResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeStreamResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeStreamResponse + switch t := that.(type) { + case *DescribeStreamResponse: + that1 = t + case DescribeStreamResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollWorkflowMessagesRequest to the protobuf v3 wire format +func (val *PollWorkflowMessagesRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollWorkflowMessagesRequest from the protobuf v3 wire format +func (val *PollWorkflowMessagesRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollWorkflowMessagesRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollWorkflowMessagesRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollWorkflowMessagesRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollWorkflowMessagesRequest + switch t := that.(type) { + case *PollWorkflowMessagesRequest: + that1 = t + case PollWorkflowMessagesRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollWorkflowMessagesResponse to the protobuf v3 wire format +func (val *PollWorkflowMessagesResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollWorkflowMessagesResponse from the protobuf v3 wire format +func (val *PollWorkflowMessagesResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollWorkflowMessagesResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollWorkflowMessagesResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollWorkflowMessagesResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollWorkflowMessagesResponse + switch t := that.(type) { + case *PollWorkflowMessagesResponse: + that1 = t + case PollWorkflowMessagesResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeWorkflowStreamRequest to the protobuf v3 wire format +func (val *DescribeWorkflowStreamRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeWorkflowStreamRequest from the protobuf v3 wire format +func (val *DescribeWorkflowStreamRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeWorkflowStreamRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeWorkflowStreamRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeWorkflowStreamRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeWorkflowStreamRequest + switch t := that.(type) { + case *DescribeWorkflowStreamRequest: + that1 = t + case DescribeWorkflowStreamRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeWorkflowStreamResponse to the protobuf v3 wire format +func (val *DescribeWorkflowStreamResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeWorkflowStreamResponse from the protobuf v3 wire format +func (val *DescribeWorkflowStreamResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeWorkflowStreamResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeWorkflowStreamResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeWorkflowStreamResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeWorkflowStreamResponse + switch t := that.(type) { + case *DescribeWorkflowStreamResponse: + that1 = t + case DescribeWorkflowStreamResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type RegisterStreamConsumerInput to the protobuf v3 wire format +func (val *RegisterStreamConsumerInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type RegisterStreamConsumerInput from the protobuf v3 wire format +func (val *RegisterStreamConsumerInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *RegisterStreamConsumerInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two RegisterStreamConsumerInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *RegisterStreamConsumerInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *RegisterStreamConsumerInput + switch t := that.(type) { + case *RegisterStreamConsumerInput: + that1 = t + case RegisterStreamConsumerInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type RegisterStreamConsumerOutput to the protobuf v3 wire format +func (val *RegisterStreamConsumerOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type RegisterStreamConsumerOutput from the protobuf v3 wire format +func (val *RegisterStreamConsumerOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *RegisterStreamConsumerOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two RegisterStreamConsumerOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *RegisterStreamConsumerOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *RegisterStreamConsumerOutput + switch t := that.(type) { + case *RegisterStreamConsumerOutput: + that1 = t + case RegisterStreamConsumerOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AdvanceConsumerHeadInput to the protobuf v3 wire format +func (val *AdvanceConsumerHeadInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AdvanceConsumerHeadInput from the protobuf v3 wire format +func (val *AdvanceConsumerHeadInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AdvanceConsumerHeadInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AdvanceConsumerHeadInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AdvanceConsumerHeadInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AdvanceConsumerHeadInput + switch t := that.(type) { + case *AdvanceConsumerHeadInput: + that1 = t + case AdvanceConsumerHeadInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AdvanceConsumerHeadOutput to the protobuf v3 wire format +func (val *AdvanceConsumerHeadOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AdvanceConsumerHeadOutput from the protobuf v3 wire format +func (val *AdvanceConsumerHeadOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AdvanceConsumerHeadOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AdvanceConsumerHeadOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AdvanceConsumerHeadOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AdvanceConsumerHeadOutput + switch t := that.(type) { + case *AdvanceConsumerHeadOutput: + that1 = t + case AdvanceConsumerHeadOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AddWorkflowMessagesRequest to the protobuf v3 wire format +func (val *AddWorkflowMessagesRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AddWorkflowMessagesRequest from the protobuf v3 wire format +func (val *AddWorkflowMessagesRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AddWorkflowMessagesRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AddWorkflowMessagesRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AddWorkflowMessagesRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AddWorkflowMessagesRequest + switch t := that.(type) { + case *AddWorkflowMessagesRequest: + that1 = t + case AddWorkflowMessagesRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AddWorkflowMessagesResponse to the protobuf v3 wire format +func (val *AddWorkflowMessagesResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AddWorkflowMessagesResponse from the protobuf v3 wire format +func (val *AddWorkflowMessagesResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AddWorkflowMessagesResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AddWorkflowMessagesResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AddWorkflowMessagesResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AddWorkflowMessagesResponse + switch t := that.(type) { + case *AddWorkflowMessagesResponse: + that1 = t + case AddWorkflowMessagesResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type RegisterStreamConsumerRequest to the protobuf v3 wire format +func (val *RegisterStreamConsumerRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type RegisterStreamConsumerRequest from the protobuf v3 wire format +func (val *RegisterStreamConsumerRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *RegisterStreamConsumerRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two RegisterStreamConsumerRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *RegisterStreamConsumerRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *RegisterStreamConsumerRequest + switch t := that.(type) { + case *RegisterStreamConsumerRequest: + that1 = t + case RegisterStreamConsumerRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type RegisterStreamConsumerResponse to the protobuf v3 wire format +func (val *RegisterStreamConsumerResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type RegisterStreamConsumerResponse from the protobuf v3 wire format +func (val *RegisterStreamConsumerResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *RegisterStreamConsumerResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two RegisterStreamConsumerResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *RegisterStreamConsumerResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *RegisterStreamConsumerResponse + switch t := that.(type) { + case *RegisterStreamConsumerResponse: + that1 = t + case RegisterStreamConsumerResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AdvanceConsumerHeadRequest to the protobuf v3 wire format +func (val *AdvanceConsumerHeadRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AdvanceConsumerHeadRequest from the protobuf v3 wire format +func (val *AdvanceConsumerHeadRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AdvanceConsumerHeadRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AdvanceConsumerHeadRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AdvanceConsumerHeadRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AdvanceConsumerHeadRequest + switch t := that.(type) { + case *AdvanceConsumerHeadRequest: + that1 = t + case AdvanceConsumerHeadRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type AdvanceConsumerHeadResponse to the protobuf v3 wire format +func (val *AdvanceConsumerHeadResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type AdvanceConsumerHeadResponse from the protobuf v3 wire format +func (val *AdvanceConsumerHeadResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *AdvanceConsumerHeadResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two AdvanceConsumerHeadResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *AdvanceConsumerHeadResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *AdvanceConsumerHeadResponse + switch t := that.(type) { + case *AdvanceConsumerHeadResponse: + that1 = t + case AdvanceConsumerHeadResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type CloseStreamRequest to the protobuf v3 wire format +func (val *CloseStreamRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CloseStreamRequest from the protobuf v3 wire format +func (val *CloseStreamRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CloseStreamRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CloseStreamRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CloseStreamRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CloseStreamRequest + switch t := that.(type) { + case *CloseStreamRequest: + that1 = t + case CloseStreamRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type CloseStreamResponse to the protobuf v3 wire format +func (val *CloseStreamResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CloseStreamResponse from the protobuf v3 wire format +func (val *CloseStreamResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CloseStreamResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CloseStreamResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CloseStreamResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CloseStreamResponse + switch t := that.(type) { + case *CloseStreamResponse: + that1 = t + case CloseStreamResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type TruncateStreamRequest to the protobuf v3 wire format +func (val *TruncateStreamRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type TruncateStreamRequest from the protobuf v3 wire format +func (val *TruncateStreamRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *TruncateStreamRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two TruncateStreamRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *TruncateStreamRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *TruncateStreamRequest + switch t := that.(type) { + case *TruncateStreamRequest: + that1 = t + case TruncateStreamRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type TruncateStreamResponse to the protobuf v3 wire format +func (val *TruncateStreamResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type TruncateStreamResponse from the protobuf v3 wire format +func (val *TruncateStreamResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *TruncateStreamResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two TruncateStreamResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *TruncateStreamResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *TruncateStreamResponse + switch t := that.(type) { + case *TruncateStreamResponse: + that1 = t + case TruncateStreamResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type ListStreamsInput to the protobuf v3 wire format +func (val *ListStreamsInput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type ListStreamsInput from the protobuf v3 wire format +func (val *ListStreamsInput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *ListStreamsInput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two ListStreamsInput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *ListStreamsInput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *ListStreamsInput + switch t := that.(type) { + case *ListStreamsInput: + that1 = t + case ListStreamsInput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type StreamListEntry to the protobuf v3 wire format +func (val *StreamListEntry) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StreamListEntry from the protobuf v3 wire format +func (val *StreamListEntry) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StreamListEntry) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StreamListEntry values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StreamListEntry) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StreamListEntry + switch t := that.(type) { + case *StreamListEntry: + that1 = t + case StreamListEntry: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type ListStreamsOutput to the protobuf v3 wire format +func (val *ListStreamsOutput) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type ListStreamsOutput from the protobuf v3 wire format +func (val *ListStreamsOutput) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *ListStreamsOutput) Size() int { + return proto.Size(val) +} + +// Equal returns whether two ListStreamsOutput values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *ListStreamsOutput) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *ListStreamsOutput + switch t := that.(type) { + case *ListStreamsOutput: + that1 = t + case ListStreamsOutput: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type ListStreamsRequest to the protobuf v3 wire format +func (val *ListStreamsRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type ListStreamsRequest from the protobuf v3 wire format +func (val *ListStreamsRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *ListStreamsRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two ListStreamsRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *ListStreamsRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *ListStreamsRequest + switch t := that.(type) { + case *ListStreamsRequest: + that1 = t + case ListStreamsRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type ListStreamsResponse to the protobuf v3 wire format +func (val *ListStreamsResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type ListStreamsResponse from the protobuf v3 wire format +func (val *ListStreamsResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *ListStreamsResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two ListStreamsResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *ListStreamsResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *ListStreamsResponse + switch t := that.(type) { + case *ListStreamsResponse: + that1 = t + case ListStreamsResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DeleteStreamRequest to the protobuf v3 wire format +func (val *DeleteStreamRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DeleteStreamRequest from the protobuf v3 wire format +func (val *DeleteStreamRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DeleteStreamRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DeleteStreamRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DeleteStreamRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DeleteStreamRequest + switch t := that.(type) { + case *DeleteStreamRequest: + that1 = t + case DeleteStreamRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DeleteStreamResponse to the protobuf v3 wire format +func (val *DeleteStreamResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DeleteStreamResponse from the protobuf v3 wire format +func (val *DeleteStreamResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DeleteStreamResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DeleteStreamResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DeleteStreamResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DeleteStreamResponse + switch t := that.(type) { + case *DeleteStreamResponse: + that1 = t + case DeleteStreamResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/chasm/lib/stream/gen/streampb/v1/request_response.pb.go b/chasm/lib/stream/gen/streampb/v1/request_response.pb.go new file mode 100644 index 00000000000..d941c63b908 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/request_response.pb.go @@ -0,0 +1,3620 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/stream/proto/v1/request_response.proto + +package streampb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + v1 "go.temporal.io/api/common/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type CreateStreamInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + Lifecycle *StreamLifecycle `protobuf:"bytes,3,opt,name=lifecycle,proto3" json:"lifecycle,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateStreamInput) Reset() { + *x = CreateStreamInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateStreamInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateStreamInput) ProtoMessage() {} + +func (x *CreateStreamInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateStreamInput.ProtoReflect.Descriptor instead. +func (*CreateStreamInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{0} +} + +func (x *CreateStreamInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *CreateStreamInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *CreateStreamInput) GetLifecycle() *StreamLifecycle { + if x != nil { + return x.Lifecycle + } + return nil +} + +type CreateStreamOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateStreamOutput) Reset() { + *x = CreateStreamOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateStreamOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateStreamOutput) ProtoMessage() {} + +func (x *CreateStreamOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateStreamOutput.ProtoReflect.Descriptor instead. +func (*CreateStreamOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{1} +} + +func (x *CreateStreamOutput) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +type AddMessagesInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + // Optional. Supplying it skips resolving the stream's current run, which is + // a persistence lookup on every call. CreateStream returns it. + RunId string `protobuf:"bytes,9,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + Messages []*StreamMessage `protobuf:"bytes,3,rep,name=messages,proto3" json:"messages,omitempty"` + // Idempotency, all optional. Supply a producer identity and sequence, or an + // expected offset, or neither and accept at-least-once. + ProducerId string `protobuf:"bytes,4,opt,name=producer_id,json=producerId,proto3" json:"producer_id,omitempty"` + Sequence int64 `protobuf:"varint,5,opt,name=sequence,proto3" json:"sequence,omitempty"` + // Guarded by use_expected_offset because proto3 optional is not supported + // by this repo's helper generator. + ExpectedOffset int64 `protobuf:"varint,6,opt,name=expected_offset,json=expectedOffset,proto3" json:"expected_offset,omitempty"` + UseExpectedOffset bool `protobuf:"varint,8,opt,name=use_expected_offset,json=useExpectedOffset,proto3" json:"use_expected_offset,omitempty"` + OwnerEpoch int64 `protobuf:"varint,7,opt,name=owner_epoch,json=ownerEpoch,proto3" json:"owner_epoch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AddMessagesInput) Reset() { + *x = AddMessagesInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AddMessagesInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddMessagesInput) ProtoMessage() {} + +func (x *AddMessagesInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddMessagesInput.ProtoReflect.Descriptor instead. +func (*AddMessagesInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{2} +} + +func (x *AddMessagesInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *AddMessagesInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *AddMessagesInput) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *AddMessagesInput) GetMessages() []*StreamMessage { + if x != nil { + return x.Messages + } + return nil +} + +func (x *AddMessagesInput) GetProducerId() string { + if x != nil { + return x.ProducerId + } + return "" +} + +func (x *AddMessagesInput) GetSequence() int64 { + if x != nil { + return x.Sequence + } + return 0 +} + +func (x *AddMessagesInput) GetExpectedOffset() int64 { + if x != nil { + return x.ExpectedOffset + } + return 0 +} + +func (x *AddMessagesInput) GetUseExpectedOffset() bool { + if x != nil { + return x.UseExpectedOffset + } + return false +} + +func (x *AddMessagesInput) GetOwnerEpoch() int64 { + if x != nil { + return x.OwnerEpoch + } + return 0 +} + +type AddMessagesOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + FirstOffset int64 `protobuf:"varint,1,opt,name=first_offset,json=firstOffset,proto3" json:"first_offset,omitempty"` + NextOffset int64 `protobuf:"varint,2,opt,name=next_offset,json=nextOffset,proto3" json:"next_offset,omitempty"` + Count int64 `protobuf:"varint,3,opt,name=count,proto3" json:"count,omitempty"` + // True when a retry matched a recorded sequence and nothing was appended. + Deduplicated bool `protobuf:"varint,4,opt,name=deduplicated,proto3" json:"deduplicated,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AddMessagesOutput) Reset() { + *x = AddMessagesOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AddMessagesOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddMessagesOutput) ProtoMessage() {} + +func (x *AddMessagesOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddMessagesOutput.ProtoReflect.Descriptor instead. +func (*AddMessagesOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{3} +} + +func (x *AddMessagesOutput) GetFirstOffset() int64 { + if x != nil { + return x.FirstOffset + } + return 0 +} + +func (x *AddMessagesOutput) GetNextOffset() int64 { + if x != nil { + return x.NextOffset + } + return 0 +} + +func (x *AddMessagesOutput) GetCount() int64 { + if x != nil { + return x.Count + } + return 0 +} + +func (x *AddMessagesOutput) GetDeduplicated() bool { + if x != nil { + return x.Deduplicated + } + return false +} + +type FinishWritingInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + ProducerId string `protobuf:"bytes,3,opt,name=producer_id,json=producerId,proto3" json:"producer_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FinishWritingInput) Reset() { + *x = FinishWritingInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FinishWritingInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FinishWritingInput) ProtoMessage() {} + +func (x *FinishWritingInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FinishWritingInput.ProtoReflect.Descriptor instead. +func (*FinishWritingInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{4} +} + +func (x *FinishWritingInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *FinishWritingInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *FinishWritingInput) GetProducerId() string { + if x != nil { + return x.ProducerId + } + return "" +} + +type FinishWritingOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FinishWritingOutput) Reset() { + *x = FinishWritingOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FinishWritingOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FinishWritingOutput) ProtoMessage() {} + +func (x *FinishWritingOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FinishWritingOutput.ProtoReflect.Descriptor instead. +func (*FinishWritingOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{5} +} + +// Registers a Workflow as a consumer of a stream it owns. The cursor lands in +// the Workflow's own state, so from then on each of its Workflow Tasks carries +// the next range and records what it consumed. +type SubscribeWorkflowInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + WorkflowId string `protobuf:"bytes,2,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"` + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + OwnerRunId string `protobuf:"bytes,6,opt,name=owner_run_id,json=ownerRunId,proto3" json:"owner_run_id,omitempty"` + // Name of the stream within the Workflow, for a stream it owns. + StreamName string `protobuf:"bytes,3,opt,name=stream_name,json=streamName,proto3" json:"stream_name,omitempty"` + // Id of a standalone stream in another execution. Exactly one of this and + // stream_name is set. + StreamId string `protobuf:"bytes,5,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + // Where to start. Resolved here rather than at delivery, so the first + // recorded range starts from a fact instead of a reading. + StartOffset int64 `protobuf:"varint,4,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribeWorkflowInput) Reset() { + *x = SubscribeWorkflowInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribeWorkflowInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribeWorkflowInput) ProtoMessage() {} + +func (x *SubscribeWorkflowInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribeWorkflowInput.ProtoReflect.Descriptor instead. +func (*SubscribeWorkflowInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{6} +} + +func (x *SubscribeWorkflowInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *SubscribeWorkflowInput) GetWorkflowId() string { + if x != nil { + return x.WorkflowId + } + return "" +} + +func (x *SubscribeWorkflowInput) GetOwnerRunId() string { + if x != nil { + return x.OwnerRunId + } + return "" +} + +func (x *SubscribeWorkflowInput) GetStreamName() string { + if x != nil { + return x.StreamName + } + return "" +} + +func (x *SubscribeWorkflowInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *SubscribeWorkflowInput) GetStartOffset() int64 { + if x != nil { + return x.StartOffset + } + return 0 +} + +type SubscribeWorkflowOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + StartOffset int64 `protobuf:"varint,1,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribeWorkflowOutput) Reset() { + *x = SubscribeWorkflowOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribeWorkflowOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribeWorkflowOutput) ProtoMessage() {} + +func (x *SubscribeWorkflowOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribeWorkflowOutput.ProtoReflect.Descriptor instead. +func (*SubscribeWorkflowOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{7} +} + +func (x *SubscribeWorkflowOutput) GetStartOffset() int64 { + if x != nil { + return x.StartOffset + } + return 0 +} + +type PollMessagesInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + // Optional, as on AddMessagesInput. + RunId string `protobuf:"bytes,7,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + FromOffset int64 `protobuf:"varint,3,opt,name=from_offset,json=fromOffset,proto3" json:"from_offset,omitempty"` + MaxMessages int32 `protobuf:"varint,4,opt,name=max_messages,json=maxMessages,proto3" json:"max_messages,omitempty"` + // Filters by exact topic. Offsets are assigned over the unfiltered stream, so + // next_offset advances past filtered-out messages too. + Topics []string `protobuf:"bytes,5,rep,name=topics,proto3" json:"topics,omitempty"` + // When set and the reader is caught up, block until something arrives, the + // stream closes, or the server's long-poll timeout elapses. A timeout returns + // an empty response rather than an error, so the caller simply polls again. + WaitNewMessages bool `protobuf:"varint,6,opt,name=wait_new_messages,json=waitNewMessages,proto3" json:"wait_new_messages,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollMessagesInput) Reset() { + *x = PollMessagesInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollMessagesInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollMessagesInput) ProtoMessage() {} + +func (x *PollMessagesInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollMessagesInput.ProtoReflect.Descriptor instead. +func (*PollMessagesInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{8} +} + +func (x *PollMessagesInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *PollMessagesInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *PollMessagesInput) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *PollMessagesInput) GetFromOffset() int64 { + if x != nil { + return x.FromOffset + } + return 0 +} + +func (x *PollMessagesInput) GetMaxMessages() int32 { + if x != nil { + return x.MaxMessages + } + return 0 +} + +func (x *PollMessagesInput) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +func (x *PollMessagesInput) GetWaitNewMessages() bool { + if x != nil { + return x.WaitNewMessages + } + return false +} + +type PollMessagesOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Messages []*StreamMessage `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"` + NextOffset int64 `protobuf:"varint,2,opt,name=next_offset,json=nextOffset,proto3" json:"next_offset,omitempty"` + HeadOffset int64 `protobuf:"varint,3,opt,name=head_offset,json=headOffset,proto3" json:"head_offset,omitempty"` + Closed bool `protobuf:"varint,4,opt,name=closed,proto3" json:"closed,omitempty"` + CloseReason *v1.Payload `protobuf:"bytes,5,opt,name=close_reason,json=closeReason,proto3" json:"close_reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollMessagesOutput) Reset() { + *x = PollMessagesOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollMessagesOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollMessagesOutput) ProtoMessage() {} + +func (x *PollMessagesOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollMessagesOutput.ProtoReflect.Descriptor instead. +func (*PollMessagesOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{9} +} + +func (x *PollMessagesOutput) GetMessages() []*StreamMessage { + if x != nil { + return x.Messages + } + return nil +} + +func (x *PollMessagesOutput) GetNextOffset() int64 { + if x != nil { + return x.NextOffset + } + return 0 +} + +func (x *PollMessagesOutput) GetHeadOffset() int64 { + if x != nil { + return x.HeadOffset + } + return 0 +} + +func (x *PollMessagesOutput) GetClosed() bool { + if x != nil { + return x.Closed + } + return false +} + +func (x *PollMessagesOutput) GetCloseReason() *v1.Payload { + if x != nil { + return x.CloseReason + } + return nil +} + +type DescribeStreamInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeStreamInput) Reset() { + *x = DescribeStreamInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeStreamInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeStreamInput) ProtoMessage() {} + +func (x *DescribeStreamInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeStreamInput.ProtoReflect.Descriptor instead. +func (*DescribeStreamInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{10} +} + +func (x *DescribeStreamInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *DescribeStreamInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +// A stream a workflow owns lives inside that workflow's execution, so it has +// no standalone id to address it by. It is named by its owner and its name +// instead, and routed on the owner. +type PollWorkflowMessagesInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + WorkflowId string `protobuf:"bytes,2,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"` + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + OwnerRunId string `protobuf:"bytes,8,opt,name=owner_run_id,json=ownerRunId,proto3" json:"owner_run_id,omitempty"` + // Empty means the workflow's default output stream. + StreamName string `protobuf:"bytes,3,opt,name=stream_name,json=streamName,proto3" json:"stream_name,omitempty"` + FromOffset int64 `protobuf:"varint,4,opt,name=from_offset,json=fromOffset,proto3" json:"from_offset,omitempty"` + MaxMessages int32 `protobuf:"varint,5,opt,name=max_messages,json=maxMessages,proto3" json:"max_messages,omitempty"` + // Filters as on PollMessagesInput. + Topics []string `protobuf:"bytes,6,rep,name=topics,proto3" json:"topics,omitempty"` + WaitNewMessages bool `protobuf:"varint,7,opt,name=wait_new_messages,json=waitNewMessages,proto3" json:"wait_new_messages,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollWorkflowMessagesInput) Reset() { + *x = PollWorkflowMessagesInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollWorkflowMessagesInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollWorkflowMessagesInput) ProtoMessage() {} + +func (x *PollWorkflowMessagesInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollWorkflowMessagesInput.ProtoReflect.Descriptor instead. +func (*PollWorkflowMessagesInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{11} +} + +func (x *PollWorkflowMessagesInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *PollWorkflowMessagesInput) GetWorkflowId() string { + if x != nil { + return x.WorkflowId + } + return "" +} + +func (x *PollWorkflowMessagesInput) GetOwnerRunId() string { + if x != nil { + return x.OwnerRunId + } + return "" +} + +func (x *PollWorkflowMessagesInput) GetStreamName() string { + if x != nil { + return x.StreamName + } + return "" +} + +func (x *PollWorkflowMessagesInput) GetFromOffset() int64 { + if x != nil { + return x.FromOffset + } + return 0 +} + +func (x *PollWorkflowMessagesInput) GetMaxMessages() int32 { + if x != nil { + return x.MaxMessages + } + return 0 +} + +func (x *PollWorkflowMessagesInput) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +func (x *PollWorkflowMessagesInput) GetWaitNewMessages() bool { + if x != nil { + return x.WaitNewMessages + } + return false +} + +type DescribeWorkflowStreamInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + WorkflowId string `protobuf:"bytes,2,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"` + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + OwnerRunId string `protobuf:"bytes,4,opt,name=owner_run_id,json=ownerRunId,proto3" json:"owner_run_id,omitempty"` + StreamName string `protobuf:"bytes,3,opt,name=stream_name,json=streamName,proto3" json:"stream_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeWorkflowStreamInput) Reset() { + *x = DescribeWorkflowStreamInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeWorkflowStreamInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeWorkflowStreamInput) ProtoMessage() {} + +func (x *DescribeWorkflowStreamInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeWorkflowStreamInput.ProtoReflect.Descriptor instead. +func (*DescribeWorkflowStreamInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{12} +} + +func (x *DescribeWorkflowStreamInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *DescribeWorkflowStreamInput) GetWorkflowId() string { + if x != nil { + return x.WorkflowId + } + return "" +} + +func (x *DescribeWorkflowStreamInput) GetOwnerRunId() string { + if x != nil { + return x.OwnerRunId + } + return "" +} + +func (x *DescribeWorkflowStreamInput) GetStreamName() string { + if x != nil { + return x.StreamName + } + return "" +} + +// Appending to a stream a workflow owns, from outside that workflow. The +// workflow's own publishes ride its Workflow Task instead. +type AddWorkflowMessagesInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + WorkflowId string `protobuf:"bytes,2,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"` + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + OwnerRunId string `protobuf:"bytes,7,opt,name=owner_run_id,json=ownerRunId,proto3" json:"owner_run_id,omitempty"` + // Empty means the workflow's default output stream. + StreamName string `protobuf:"bytes,3,opt,name=stream_name,json=streamName,proto3" json:"stream_name,omitempty"` + Messages []*StreamMessage `protobuf:"bytes,4,rep,name=messages,proto3" json:"messages,omitempty"` + // Optional idempotency, as on AddMessagesInput. + ProducerId string `protobuf:"bytes,5,opt,name=producer_id,json=producerId,proto3" json:"producer_id,omitempty"` + Sequence int64 `protobuf:"varint,6,opt,name=sequence,proto3" json:"sequence,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AddWorkflowMessagesInput) Reset() { + *x = AddWorkflowMessagesInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AddWorkflowMessagesInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddWorkflowMessagesInput) ProtoMessage() {} + +func (x *AddWorkflowMessagesInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddWorkflowMessagesInput.ProtoReflect.Descriptor instead. +func (*AddWorkflowMessagesInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{13} +} + +func (x *AddWorkflowMessagesInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *AddWorkflowMessagesInput) GetWorkflowId() string { + if x != nil { + return x.WorkflowId + } + return "" +} + +func (x *AddWorkflowMessagesInput) GetOwnerRunId() string { + if x != nil { + return x.OwnerRunId + } + return "" +} + +func (x *AddWorkflowMessagesInput) GetStreamName() string { + if x != nil { + return x.StreamName + } + return "" +} + +func (x *AddWorkflowMessagesInput) GetMessages() []*StreamMessage { + if x != nil { + return x.Messages + } + return nil +} + +func (x *AddWorkflowMessagesInput) GetProducerId() string { + if x != nil { + return x.ProducerId + } + return "" +} + +func (x *AddWorkflowMessagesInput) GetSequence() int64 { + if x != nil { + return x.Sequence + } + return 0 +} + +type DescribeStreamOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + State *StreamState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeStreamOutput) Reset() { + *x = DescribeStreamOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeStreamOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeStreamOutput) ProtoMessage() {} + +func (x *DescribeStreamOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeStreamOutput.ProtoReflect.Descriptor instead. +func (*DescribeStreamOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{14} +} + +func (x *DescribeStreamOutput) GetState() *StreamState { + if x != nil { + return x.State + } + return nil +} + +type CloseStreamInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + Reason *v1.Payload `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CloseStreamInput) Reset() { + *x = CloseStreamInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CloseStreamInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloseStreamInput) ProtoMessage() {} + +func (x *CloseStreamInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloseStreamInput.ProtoReflect.Descriptor instead. +func (*CloseStreamInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{15} +} + +func (x *CloseStreamInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *CloseStreamInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *CloseStreamInput) GetReason() *v1.Payload { + if x != nil { + return x.Reason + } + return nil +} + +type CloseStreamOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CloseStreamOutput) Reset() { + *x = CloseStreamOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CloseStreamOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloseStreamOutput) ProtoMessage() {} + +func (x *CloseStreamOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloseStreamOutput.ProtoReflect.Descriptor instead. +func (*CloseStreamOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{16} +} + +type TruncateStreamInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + NewBaseOffset int64 `protobuf:"varint,3,opt,name=new_base_offset,json=newBaseOffset,proto3" json:"new_base_offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TruncateStreamInput) Reset() { + *x = TruncateStreamInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TruncateStreamInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TruncateStreamInput) ProtoMessage() {} + +func (x *TruncateStreamInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TruncateStreamInput.ProtoReflect.Descriptor instead. +func (*TruncateStreamInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{17} +} + +func (x *TruncateStreamInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *TruncateStreamInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *TruncateStreamInput) GetNewBaseOffset() int64 { + if x != nil { + return x.NewBaseOffset + } + return 0 +} + +type TruncateStreamOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TruncateStreamOutput) Reset() { + *x = TruncateStreamOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TruncateStreamOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TruncateStreamOutput) ProtoMessage() {} + +func (x *TruncateStreamOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TruncateStreamOutput.ProtoReflect.Descriptor instead. +func (*TruncateStreamOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{18} +} + +type DeleteStreamInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteStreamInput) Reset() { + *x = DeleteStreamInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteStreamInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteStreamInput) ProtoMessage() {} + +func (x *DeleteStreamInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteStreamInput.ProtoReflect.Descriptor instead. +func (*DeleteStreamInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{19} +} + +func (x *DeleteStreamInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *DeleteStreamInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +type DeleteStreamOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteStreamOutput) Reset() { + *x = DeleteStreamOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteStreamOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteStreamOutput) ProtoMessage() {} + +func (x *DeleteStreamOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteStreamOutput.ProtoReflect.Descriptor instead. +func (*DeleteStreamOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{20} +} + +type CreateStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *CreateStreamInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateStreamRequest) Reset() { + *x = CreateStreamRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateStreamRequest) ProtoMessage() {} + +func (x *CreateStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateStreamRequest.ProtoReflect.Descriptor instead. +func (*CreateStreamRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{21} +} + +func (x *CreateStreamRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *CreateStreamRequest) GetFrontendRequest() *CreateStreamInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type CreateStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *CreateStreamOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateStreamResponse) Reset() { + *x = CreateStreamResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateStreamResponse) ProtoMessage() {} + +func (x *CreateStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateStreamResponse.ProtoReflect.Descriptor instead. +func (*CreateStreamResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{22} +} + +func (x *CreateStreamResponse) GetFrontendResponse() *CreateStreamOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type AddMessagesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *AddMessagesInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AddMessagesRequest) Reset() { + *x = AddMessagesRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AddMessagesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddMessagesRequest) ProtoMessage() {} + +func (x *AddMessagesRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddMessagesRequest.ProtoReflect.Descriptor instead. +func (*AddMessagesRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{23} +} + +func (x *AddMessagesRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *AddMessagesRequest) GetFrontendRequest() *AddMessagesInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type AddMessagesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *AddMessagesOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AddMessagesResponse) Reset() { + *x = AddMessagesResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AddMessagesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddMessagesResponse) ProtoMessage() {} + +func (x *AddMessagesResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddMessagesResponse.ProtoReflect.Descriptor instead. +func (*AddMessagesResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{24} +} + +func (x *AddMessagesResponse) GetFrontendResponse() *AddMessagesOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type FinishWritingRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *FinishWritingInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FinishWritingRequest) Reset() { + *x = FinishWritingRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FinishWritingRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FinishWritingRequest) ProtoMessage() {} + +func (x *FinishWritingRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FinishWritingRequest.ProtoReflect.Descriptor instead. +func (*FinishWritingRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{25} +} + +func (x *FinishWritingRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *FinishWritingRequest) GetFrontendRequest() *FinishWritingInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type FinishWritingResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *FinishWritingOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FinishWritingResponse) Reset() { + *x = FinishWritingResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FinishWritingResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FinishWritingResponse) ProtoMessage() {} + +func (x *FinishWritingResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FinishWritingResponse.ProtoReflect.Descriptor instead. +func (*FinishWritingResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{26} +} + +func (x *FinishWritingResponse) GetFrontendResponse() *FinishWritingOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type SubscribeWorkflowRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *SubscribeWorkflowInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribeWorkflowRequest) Reset() { + *x = SubscribeWorkflowRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribeWorkflowRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribeWorkflowRequest) ProtoMessage() {} + +func (x *SubscribeWorkflowRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribeWorkflowRequest.ProtoReflect.Descriptor instead. +func (*SubscribeWorkflowRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{27} +} + +func (x *SubscribeWorkflowRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *SubscribeWorkflowRequest) GetFrontendRequest() *SubscribeWorkflowInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type SubscribeWorkflowResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *SubscribeWorkflowOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribeWorkflowResponse) Reset() { + *x = SubscribeWorkflowResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribeWorkflowResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribeWorkflowResponse) ProtoMessage() {} + +func (x *SubscribeWorkflowResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribeWorkflowResponse.ProtoReflect.Descriptor instead. +func (*SubscribeWorkflowResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{28} +} + +func (x *SubscribeWorkflowResponse) GetFrontendResponse() *SubscribeWorkflowOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type PollMessagesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *PollMessagesInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollMessagesRequest) Reset() { + *x = PollMessagesRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollMessagesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollMessagesRequest) ProtoMessage() {} + +func (x *PollMessagesRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollMessagesRequest.ProtoReflect.Descriptor instead. +func (*PollMessagesRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{29} +} + +func (x *PollMessagesRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *PollMessagesRequest) GetFrontendRequest() *PollMessagesInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type PollMessagesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *PollMessagesOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollMessagesResponse) Reset() { + *x = PollMessagesResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollMessagesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollMessagesResponse) ProtoMessage() {} + +func (x *PollMessagesResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollMessagesResponse.ProtoReflect.Descriptor instead. +func (*PollMessagesResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{30} +} + +func (x *PollMessagesResponse) GetFrontendResponse() *PollMessagesOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type DescribeStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *DescribeStreamInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeStreamRequest) Reset() { + *x = DescribeStreamRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeStreamRequest) ProtoMessage() {} + +func (x *DescribeStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[31] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeStreamRequest.ProtoReflect.Descriptor instead. +func (*DescribeStreamRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{31} +} + +func (x *DescribeStreamRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *DescribeStreamRequest) GetFrontendRequest() *DescribeStreamInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type DescribeStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *DescribeStreamOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeStreamResponse) Reset() { + *x = DescribeStreamResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeStreamResponse) ProtoMessage() {} + +func (x *DescribeStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[32] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeStreamResponse.ProtoReflect.Descriptor instead. +func (*DescribeStreamResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{32} +} + +func (x *DescribeStreamResponse) GetFrontendResponse() *DescribeStreamOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type PollWorkflowMessagesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *PollWorkflowMessagesInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollWorkflowMessagesRequest) Reset() { + *x = PollWorkflowMessagesRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollWorkflowMessagesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollWorkflowMessagesRequest) ProtoMessage() {} + +func (x *PollWorkflowMessagesRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollWorkflowMessagesRequest.ProtoReflect.Descriptor instead. +func (*PollWorkflowMessagesRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{33} +} + +func (x *PollWorkflowMessagesRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *PollWorkflowMessagesRequest) GetFrontendRequest() *PollWorkflowMessagesInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type PollWorkflowMessagesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *PollMessagesOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollWorkflowMessagesResponse) Reset() { + *x = PollWorkflowMessagesResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollWorkflowMessagesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollWorkflowMessagesResponse) ProtoMessage() {} + +func (x *PollWorkflowMessagesResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollWorkflowMessagesResponse.ProtoReflect.Descriptor instead. +func (*PollWorkflowMessagesResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{34} +} + +func (x *PollWorkflowMessagesResponse) GetFrontendResponse() *PollMessagesOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type DescribeWorkflowStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *DescribeWorkflowStreamInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeWorkflowStreamRequest) Reset() { + *x = DescribeWorkflowStreamRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeWorkflowStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeWorkflowStreamRequest) ProtoMessage() {} + +func (x *DescribeWorkflowStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[35] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeWorkflowStreamRequest.ProtoReflect.Descriptor instead. +func (*DescribeWorkflowStreamRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{35} +} + +func (x *DescribeWorkflowStreamRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *DescribeWorkflowStreamRequest) GetFrontendRequest() *DescribeWorkflowStreamInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type DescribeWorkflowStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *DescribeStreamOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeWorkflowStreamResponse) Reset() { + *x = DescribeWorkflowStreamResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeWorkflowStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeWorkflowStreamResponse) ProtoMessage() {} + +func (x *DescribeWorkflowStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeWorkflowStreamResponse.ProtoReflect.Descriptor instead. +func (*DescribeWorkflowStreamResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{36} +} + +func (x *DescribeWorkflowStreamResponse) GetFrontendResponse() *DescribeStreamOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +// Registering a consumer on a stream in another execution. Split out from +// SubscribeWorkflow because the two halves live on different shards: the pin +// goes on the stream, the cursor goes on the consuming workflow, and a handler +// can only reach the shard it was routed to. +type RegisterStreamConsumerInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + // The workflow that will consume, which names the pin. + ConsumerWorkflowId string `protobuf:"bytes,3,opt,name=consumer_workflow_id,json=consumerWorkflowId,proto3" json:"consumer_workflow_id,omitempty"` + // Negative means from wherever the stream is when the pin is taken. Resolved + // here, where the frontier is, and returned so the cursor records a fact. + StartOffset int64 `protobuf:"varint,4,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RegisterStreamConsumerInput) Reset() { + *x = RegisterStreamConsumerInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RegisterStreamConsumerInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterStreamConsumerInput) ProtoMessage() {} + +func (x *RegisterStreamConsumerInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[37] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterStreamConsumerInput.ProtoReflect.Descriptor instead. +func (*RegisterStreamConsumerInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{37} +} + +func (x *RegisterStreamConsumerInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *RegisterStreamConsumerInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *RegisterStreamConsumerInput) GetConsumerWorkflowId() string { + if x != nil { + return x.ConsumerWorkflowId + } + return "" +} + +func (x *RegisterStreamConsumerInput) GetStartOffset() int64 { + if x != nil { + return x.StartOffset + } + return 0 +} + +type RegisterStreamConsumerOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + StartOffset int64 `protobuf:"varint,1,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"` + // What the consumer needs to address the log, all decided by the stream. + CollectionId string `protobuf:"bytes,2,opt,name=collection_id,json=collectionId,proto3" json:"collection_id,omitempty"` + BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` + KnownHead int64 `protobuf:"varint,4,opt,name=known_head,json=knownHead,proto3" json:"known_head,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RegisterStreamConsumerOutput) Reset() { + *x = RegisterStreamConsumerOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RegisterStreamConsumerOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterStreamConsumerOutput) ProtoMessage() {} + +func (x *RegisterStreamConsumerOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[38] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterStreamConsumerOutput.ProtoReflect.Descriptor instead. +func (*RegisterStreamConsumerOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{38} +} + +func (x *RegisterStreamConsumerOutput) GetStartOffset() int64 { + if x != nil { + return x.StartOffset + } + return 0 +} + +func (x *RegisterStreamConsumerOutput) GetCollectionId() string { + if x != nil { + return x.CollectionId + } + return "" +} + +func (x *RegisterStreamConsumerOutput) GetBucketSize() int64 { + if x != nil { + return x.BucketSize + } + return 0 +} + +func (x *RegisterStreamConsumerOutput) GetKnownHead() int64 { + if x != nil { + return x.KnownHead + } + return 0 +} + +// Telling one consumer that the frontier moved. Routed to the consumer, which +// is not where the stream lives. +type AdvanceConsumerHeadInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + WorkflowId string `protobuf:"bytes,2,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"` + // The run that subscribed. A successor from continue-as-new holds no cursor + // for this stream, so pushing the frontier at it would land nowhere. + OwnerRunId string `protobuf:"bytes,5,opt,name=owner_run_id,json=ownerRunId,proto3" json:"owner_run_id,omitempty"` + StreamId string `protobuf:"bytes,3,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + HeadOffset int64 `protobuf:"varint,4,opt,name=head_offset,json=headOffset,proto3" json:"head_offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AdvanceConsumerHeadInput) Reset() { + *x = AdvanceConsumerHeadInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[39] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AdvanceConsumerHeadInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AdvanceConsumerHeadInput) ProtoMessage() {} + +func (x *AdvanceConsumerHeadInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[39] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AdvanceConsumerHeadInput.ProtoReflect.Descriptor instead. +func (*AdvanceConsumerHeadInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{39} +} + +func (x *AdvanceConsumerHeadInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *AdvanceConsumerHeadInput) GetWorkflowId() string { + if x != nil { + return x.WorkflowId + } + return "" +} + +func (x *AdvanceConsumerHeadInput) GetOwnerRunId() string { + if x != nil { + return x.OwnerRunId + } + return "" +} + +func (x *AdvanceConsumerHeadInput) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *AdvanceConsumerHeadInput) GetHeadOffset() int64 { + if x != nil { + return x.HeadOffset + } + return 0 +} + +type AdvanceConsumerHeadOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AdvanceConsumerHeadOutput) Reset() { + *x = AdvanceConsumerHeadOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[40] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AdvanceConsumerHeadOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AdvanceConsumerHeadOutput) ProtoMessage() {} + +func (x *AdvanceConsumerHeadOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[40] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AdvanceConsumerHeadOutput.ProtoReflect.Descriptor instead. +func (*AdvanceConsumerHeadOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{40} +} + +type AddWorkflowMessagesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *AddWorkflowMessagesInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AddWorkflowMessagesRequest) Reset() { + *x = AddWorkflowMessagesRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[41] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AddWorkflowMessagesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddWorkflowMessagesRequest) ProtoMessage() {} + +func (x *AddWorkflowMessagesRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[41] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddWorkflowMessagesRequest.ProtoReflect.Descriptor instead. +func (*AddWorkflowMessagesRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{41} +} + +func (x *AddWorkflowMessagesRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *AddWorkflowMessagesRequest) GetFrontendRequest() *AddWorkflowMessagesInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type AddWorkflowMessagesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *AddMessagesOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AddWorkflowMessagesResponse) Reset() { + *x = AddWorkflowMessagesResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[42] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AddWorkflowMessagesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddWorkflowMessagesResponse) ProtoMessage() {} + +func (x *AddWorkflowMessagesResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[42] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddWorkflowMessagesResponse.ProtoReflect.Descriptor instead. +func (*AddWorkflowMessagesResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{42} +} + +func (x *AddWorkflowMessagesResponse) GetFrontendResponse() *AddMessagesOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type RegisterStreamConsumerRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *RegisterStreamConsumerInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RegisterStreamConsumerRequest) Reset() { + *x = RegisterStreamConsumerRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[43] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RegisterStreamConsumerRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterStreamConsumerRequest) ProtoMessage() {} + +func (x *RegisterStreamConsumerRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[43] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterStreamConsumerRequest.ProtoReflect.Descriptor instead. +func (*RegisterStreamConsumerRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{43} +} + +func (x *RegisterStreamConsumerRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *RegisterStreamConsumerRequest) GetFrontendRequest() *RegisterStreamConsumerInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type RegisterStreamConsumerResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *RegisterStreamConsumerOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RegisterStreamConsumerResponse) Reset() { + *x = RegisterStreamConsumerResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[44] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RegisterStreamConsumerResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterStreamConsumerResponse) ProtoMessage() {} + +func (x *RegisterStreamConsumerResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[44] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterStreamConsumerResponse.ProtoReflect.Descriptor instead. +func (*RegisterStreamConsumerResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{44} +} + +func (x *RegisterStreamConsumerResponse) GetFrontendResponse() *RegisterStreamConsumerOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type AdvanceConsumerHeadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *AdvanceConsumerHeadInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AdvanceConsumerHeadRequest) Reset() { + *x = AdvanceConsumerHeadRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[45] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AdvanceConsumerHeadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AdvanceConsumerHeadRequest) ProtoMessage() {} + +func (x *AdvanceConsumerHeadRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[45] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AdvanceConsumerHeadRequest.ProtoReflect.Descriptor instead. +func (*AdvanceConsumerHeadRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{45} +} + +func (x *AdvanceConsumerHeadRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *AdvanceConsumerHeadRequest) GetFrontendRequest() *AdvanceConsumerHeadInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type AdvanceConsumerHeadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *AdvanceConsumerHeadOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AdvanceConsumerHeadResponse) Reset() { + *x = AdvanceConsumerHeadResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[46] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AdvanceConsumerHeadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AdvanceConsumerHeadResponse) ProtoMessage() {} + +func (x *AdvanceConsumerHeadResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[46] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AdvanceConsumerHeadResponse.ProtoReflect.Descriptor instead. +func (*AdvanceConsumerHeadResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{46} +} + +func (x *AdvanceConsumerHeadResponse) GetFrontendResponse() *AdvanceConsumerHeadOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type CloseStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *CloseStreamInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CloseStreamRequest) Reset() { + *x = CloseStreamRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[47] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CloseStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloseStreamRequest) ProtoMessage() {} + +func (x *CloseStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[47] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloseStreamRequest.ProtoReflect.Descriptor instead. +func (*CloseStreamRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{47} +} + +func (x *CloseStreamRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *CloseStreamRequest) GetFrontendRequest() *CloseStreamInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type CloseStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *CloseStreamOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CloseStreamResponse) Reset() { + *x = CloseStreamResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CloseStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloseStreamResponse) ProtoMessage() {} + +func (x *CloseStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[48] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloseStreamResponse.ProtoReflect.Descriptor instead. +func (*CloseStreamResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{48} +} + +func (x *CloseStreamResponse) GetFrontendResponse() *CloseStreamOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type TruncateStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *TruncateStreamInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TruncateStreamRequest) Reset() { + *x = TruncateStreamRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TruncateStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TruncateStreamRequest) ProtoMessage() {} + +func (x *TruncateStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[49] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TruncateStreamRequest.ProtoReflect.Descriptor instead. +func (*TruncateStreamRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{49} +} + +func (x *TruncateStreamRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *TruncateStreamRequest) GetFrontendRequest() *TruncateStreamInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type TruncateStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *TruncateStreamOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TruncateStreamResponse) Reset() { + *x = TruncateStreamResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[50] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TruncateStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TruncateStreamResponse) ProtoMessage() {} + +func (x *TruncateStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[50] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TruncateStreamResponse.ProtoReflect.Descriptor instead. +func (*TruncateStreamResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{50} +} + +func (x *TruncateStreamResponse) GetFrontendResponse() *TruncateStreamOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type ListStreamsInput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + PageSize int32 `protobuf:"varint,2,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + NextPageToken []byte `protobuf:"bytes,3,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` + Query string `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListStreamsInput) Reset() { + *x = ListStreamsInput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[51] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListStreamsInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListStreamsInput) ProtoMessage() {} + +func (x *ListStreamsInput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[51] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListStreamsInput.ProtoReflect.Descriptor instead. +func (*ListStreamsInput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{51} +} + +func (x *ListStreamsInput) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *ListStreamsInput) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *ListStreamsInput) GetNextPageToken() []byte { + if x != nil { + return x.NextPageToken + } + return nil +} + +func (x *ListStreamsInput) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +type StreamListEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + StreamId string `protobuf:"bytes,1,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + RunId string `protobuf:"bytes,2,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamListEntry) Reset() { + *x = StreamListEntry{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[52] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamListEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamListEntry) ProtoMessage() {} + +func (x *StreamListEntry) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[52] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamListEntry.ProtoReflect.Descriptor instead. +func (*StreamListEntry) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{52} +} + +func (x *StreamListEntry) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *StreamListEntry) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +type ListStreamsOutput struct { + state protoimpl.MessageState `protogen:"open.v1"` + Streams []*StreamListEntry `protobuf:"bytes,1,rep,name=streams,proto3" json:"streams,omitempty"` + NextPageToken []byte `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListStreamsOutput) Reset() { + *x = ListStreamsOutput{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[53] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListStreamsOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListStreamsOutput) ProtoMessage() {} + +func (x *ListStreamsOutput) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[53] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListStreamsOutput.ProtoReflect.Descriptor instead. +func (*ListStreamsOutput) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{53} +} + +func (x *ListStreamsOutput) GetStreams() []*StreamListEntry { + if x != nil { + return x.Streams + } + return nil +} + +func (x *ListStreamsOutput) GetNextPageToken() []byte { + if x != nil { + return x.NextPageToken + } + return nil +} + +type ListStreamsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *ListStreamsInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListStreamsRequest) Reset() { + *x = ListStreamsRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListStreamsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListStreamsRequest) ProtoMessage() {} + +func (x *ListStreamsRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[54] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListStreamsRequest.ProtoReflect.Descriptor instead. +func (*ListStreamsRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{54} +} + +func (x *ListStreamsRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *ListStreamsRequest) GetFrontendRequest() *ListStreamsInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type ListStreamsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *ListStreamsOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListStreamsResponse) Reset() { + *x = ListStreamsResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[55] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListStreamsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListStreamsResponse) ProtoMessage() {} + +func (x *ListStreamsResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[55] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListStreamsResponse.ProtoReflect.Descriptor instead. +func (*ListStreamsResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{55} +} + +func (x *ListStreamsResponse) GetFrontendResponse() *ListStreamsOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type DeleteStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *DeleteStreamInput `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteStreamRequest) Reset() { + *x = DeleteStreamRequest{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[56] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteStreamRequest) ProtoMessage() {} + +func (x *DeleteStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[56] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteStreamRequest.ProtoReflect.Descriptor instead. +func (*DeleteStreamRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{56} +} + +func (x *DeleteStreamRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *DeleteStreamRequest) GetFrontendRequest() *DeleteStreamInput { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type DeleteStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *DeleteStreamOutput `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteStreamResponse) Reset() { + *x = DeleteStreamResponse{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[57] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteStreamResponse) ProtoMessage() {} + +func (x *DeleteStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes[57] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteStreamResponse.ProtoReflect.Descriptor instead. +func (*DeleteStreamResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDescGZIP(), []int{57} +} + +func (x *DeleteStreamResponse) GetFrontendResponse() *DeleteStreamOutput { + if x != nil { + return x.FrontendResponse + } + return nil +} + +var File_temporal_server_chasm_lib_stream_proto_v1_request_response_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDesc = "" + + "\n" + + "@temporal/server/chasm/lib/stream/proto/v1/request_response.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a7temporal/server/chasm/lib/stream/proto/v1/message.proto\x1a.temporal.server.chasm.lib.stream.proto.v1.FinishWritingOutputR\x10frontendResponse\"\xab\x01\n" + + "\x18SubscribeWorkflowRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12l\n" + + "\x10frontend_request\x18\x02 \x01(\v2A.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowInputR\x0ffrontendRequest\"\x8c\x01\n" + + "\x19SubscribeWorkflowResponse\x12o\n" + + "\x11frontend_response\x18\x01 \x01(\v2B.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowOutputR\x10frontendResponse\"\xa1\x01\n" + + "\x13PollMessagesRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12g\n" + + "\x10frontend_request\x18\x02 \x01(\v2<.temporal.server.chasm.lib.stream.proto.v1.PollMessagesInputR\x0ffrontendRequest\"\x82\x01\n" + + "\x14PollMessagesResponse\x12j\n" + + "\x11frontend_response\x18\x01 \x01(\v2=.temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutputR\x10frontendResponse\"\xa5\x01\n" + + "\x15DescribeStreamRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12i\n" + + "\x10frontend_request\x18\x02 \x01(\v2>.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamInputR\x0ffrontendRequest\"\x86\x01\n" + + "\x16DescribeStreamResponse\x12l\n" + + "\x11frontend_response\x18\x01 \x01(\v2?.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutputR\x10frontendResponse\"\xb1\x01\n" + + "\x1bPollWorkflowMessagesRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12o\n" + + "\x10frontend_request\x18\x02 \x01(\v2D.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesInputR\x0ffrontendRequest\"\x8a\x01\n" + + "\x1cPollWorkflowMessagesResponse\x12j\n" + + "\x11frontend_response\x18\x01 \x01(\v2=.temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutputR\x10frontendResponse\"\xb5\x01\n" + + "\x1dDescribeWorkflowStreamRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12q\n" + + "\x10frontend_request\x18\x02 \x01(\v2F.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamInputR\x0ffrontendRequest\"\x8e\x01\n" + + "\x1eDescribeWorkflowStreamResponse\x12l\n" + + "\x11frontend_response\x18\x01 \x01(\v2?.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutputR\x10frontendResponse\"\xad\x01\n" + + "\x1bRegisterStreamConsumerInput\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12\x1b\n" + + "\tstream_id\x18\x02 \x01(\tR\bstreamId\x120\n" + + "\x14consumer_workflow_id\x18\x03 \x01(\tR\x12consumerWorkflowId\x12!\n" + + "\fstart_offset\x18\x04 \x01(\x03R\vstartOffset\"\xa6\x01\n" + + "\x1cRegisterStreamConsumerOutput\x12!\n" + + "\fstart_offset\x18\x01 \x01(\x03R\vstartOffset\x12#\n" + + "\rcollection_id\x18\x02 \x01(\tR\fcollectionId\x12\x1f\n" + + "\vbucket_size\x18\x03 \x01(\x03R\n" + + "bucketSize\x12\x1d\n" + + "\n" + + "known_head\x18\x04 \x01(\x03R\tknownHead\"\xb9\x01\n" + + "\x18AdvanceConsumerHeadInput\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12\x1f\n" + + "\vworkflow_id\x18\x02 \x01(\tR\n" + + "workflowId\x12 \n" + + "\fowner_run_id\x18\x05 \x01(\tR\n" + + "ownerRunId\x12\x1b\n" + + "\tstream_id\x18\x03 \x01(\tR\bstreamId\x12\x1f\n" + + "\vhead_offset\x18\x04 \x01(\x03R\n" + + "headOffset\"\x1b\n" + + "\x19AdvanceConsumerHeadOutput\"\xaf\x01\n" + + "\x1aAddWorkflowMessagesRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12n\n" + + "\x10frontend_request\x18\x02 \x01(\v2C.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesInputR\x0ffrontendRequest\"\x88\x01\n" + + "\x1bAddWorkflowMessagesResponse\x12i\n" + + "\x11frontend_response\x18\x01 \x01(\v2<.temporal.server.chasm.lib.stream.proto.v1.AddMessagesOutputR\x10frontendResponse\"\xb5\x01\n" + + "\x1dRegisterStreamConsumerRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12q\n" + + "\x10frontend_request\x18\x02 \x01(\v2F.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerInputR\x0ffrontendRequest\"\x96\x01\n" + + "\x1eRegisterStreamConsumerResponse\x12t\n" + + "\x11frontend_response\x18\x01 \x01(\v2G.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerOutputR\x10frontendResponse\"\xaf\x01\n" + + "\x1aAdvanceConsumerHeadRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12n\n" + + "\x10frontend_request\x18\x02 \x01(\v2C.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadInputR\x0ffrontendRequest\"\x90\x01\n" + + "\x1bAdvanceConsumerHeadResponse\x12q\n" + + "\x11frontend_response\x18\x01 \x01(\v2D.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadOutputR\x10frontendResponse\"\x9f\x01\n" + + "\x12CloseStreamRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12f\n" + + "\x10frontend_request\x18\x02 \x01(\v2;.temporal.server.chasm.lib.stream.proto.v1.CloseStreamInputR\x0ffrontendRequest\"\x80\x01\n" + + "\x13CloseStreamResponse\x12i\n" + + "\x11frontend_response\x18\x01 \x01(\v2<.temporal.server.chasm.lib.stream.proto.v1.CloseStreamOutputR\x10frontendResponse\"\xa5\x01\n" + + "\x15TruncateStreamRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12i\n" + + "\x10frontend_request\x18\x02 \x01(\v2>.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamInputR\x0ffrontendRequest\"\x86\x01\n" + + "\x16TruncateStreamResponse\x12l\n" + + "\x11frontend_response\x18\x01 \x01(\v2?.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamOutputR\x10frontendResponse\"\x8b\x01\n" + + "\x10ListStreamsInput\x12\x1c\n" + + "\tnamespace\x18\x01 \x01(\tR\tnamespace\x12\x1b\n" + + "\tpage_size\x18\x02 \x01(\x05R\bpageSize\x12&\n" + + "\x0fnext_page_token\x18\x03 \x01(\fR\rnextPageToken\x12\x14\n" + + "\x05query\x18\x04 \x01(\tR\x05query\"E\n" + + "\x0fStreamListEntry\x12\x1b\n" + + "\tstream_id\x18\x01 \x01(\tR\bstreamId\x12\x15\n" + + "\x06run_id\x18\x02 \x01(\tR\x05runId\"\x91\x01\n" + + "\x11ListStreamsOutput\x12T\n" + + "\astreams\x18\x01 \x03(\v2:.temporal.server.chasm.lib.stream.proto.v1.StreamListEntryR\astreams\x12&\n" + + "\x0fnext_page_token\x18\x02 \x01(\fR\rnextPageToken\"\x9f\x01\n" + + "\x12ListStreamsRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12f\n" + + "\x10frontend_request\x18\x02 \x01(\v2;.temporal.server.chasm.lib.stream.proto.v1.ListStreamsInputR\x0ffrontendRequest\"\x80\x01\n" + + "\x13ListStreamsResponse\x12i\n" + + "\x11frontend_response\x18\x01 \x01(\v2<.temporal.server.chasm.lib.stream.proto.v1.ListStreamsOutputR\x10frontendResponse\"\xa1\x01\n" + + "\x13DeleteStreamRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12g\n" + + "\x10frontend_request\x18\x02 \x01(\v2<.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamInputR\x0ffrontendRequest\"\x82\x01\n" + + "\x14DeleteStreamResponse\x12j\n" + + "\x11frontend_response\x18\x01 \x01(\v2=.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamOutputR\x10frontendResponseB>Z temporal.server.chasm.lib.stream.proto.v1.StreamLifecycle + 59, // 1: temporal.server.chasm.lib.stream.proto.v1.AddMessagesInput.messages:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamMessage + 59, // 2: temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput.messages:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamMessage + 60, // 3: temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput.close_reason:type_name -> temporal.api.common.v1.Payload + 59, // 4: temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesInput.messages:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamMessage + 61, // 5: temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutput.state:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamState + 60, // 6: temporal.server.chasm.lib.stream.proto.v1.CloseStreamInput.reason:type_name -> temporal.api.common.v1.Payload + 0, // 7: temporal.server.chasm.lib.stream.proto.v1.CreateStreamRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.CreateStreamInput + 1, // 8: temporal.server.chasm.lib.stream.proto.v1.CreateStreamResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.CreateStreamOutput + 2, // 9: temporal.server.chasm.lib.stream.proto.v1.AddMessagesRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.AddMessagesInput + 3, // 10: temporal.server.chasm.lib.stream.proto.v1.AddMessagesResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.AddMessagesOutput + 4, // 11: temporal.server.chasm.lib.stream.proto.v1.FinishWritingRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.FinishWritingInput + 5, // 12: temporal.server.chasm.lib.stream.proto.v1.FinishWritingResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.FinishWritingOutput + 6, // 13: temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowInput + 7, // 14: temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowOutput + 8, // 15: temporal.server.chasm.lib.stream.proto.v1.PollMessagesRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.PollMessagesInput + 9, // 16: temporal.server.chasm.lib.stream.proto.v1.PollMessagesResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput + 10, // 17: temporal.server.chasm.lib.stream.proto.v1.DescribeStreamRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.DescribeStreamInput + 14, // 18: temporal.server.chasm.lib.stream.proto.v1.DescribeStreamResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutput + 11, // 19: temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesInput + 9, // 20: temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.PollMessagesOutput + 12, // 21: temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamInput + 14, // 22: temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.DescribeStreamOutput + 13, // 23: temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesInput + 3, // 24: temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.AddMessagesOutput + 37, // 25: temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerInput + 38, // 26: temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerOutput + 39, // 27: temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadInput + 40, // 28: temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadOutput + 15, // 29: temporal.server.chasm.lib.stream.proto.v1.CloseStreamRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.CloseStreamInput + 16, // 30: temporal.server.chasm.lib.stream.proto.v1.CloseStreamResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.CloseStreamOutput + 17, // 31: temporal.server.chasm.lib.stream.proto.v1.TruncateStreamRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.TruncateStreamInput + 18, // 32: temporal.server.chasm.lib.stream.proto.v1.TruncateStreamResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.TruncateStreamOutput + 52, // 33: temporal.server.chasm.lib.stream.proto.v1.ListStreamsOutput.streams:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamListEntry + 51, // 34: temporal.server.chasm.lib.stream.proto.v1.ListStreamsRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.ListStreamsInput + 53, // 35: temporal.server.chasm.lib.stream.proto.v1.ListStreamsResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.ListStreamsOutput + 19, // 36: temporal.server.chasm.lib.stream.proto.v1.DeleteStreamRequest.frontend_request:type_name -> temporal.server.chasm.lib.stream.proto.v1.DeleteStreamInput + 20, // 37: temporal.server.chasm.lib.stream.proto.v1.DeleteStreamResponse.frontend_response:type_name -> temporal.server.chasm.lib.stream.proto.v1.DeleteStreamOutput + 38, // [38:38] is the sub-list for method output_type + 38, // [38:38] is the sub-list for method input_type + 38, // [38:38] is the sub-list for extension type_name + 38, // [38:38] is the sub-list for extension extendee + 0, // [0:38] is the sub-list for field type_name +} + +func init() { file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_init() } +func file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_init() { + if File_temporal_server_chasm_lib_stream_proto_v1_request_response_proto != nil { + return + } + file_temporal_server_chasm_lib_stream_proto_v1_message_proto_init() + file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDesc), len(file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_rawDesc)), + NumEnums: 0, + NumMessages: 58, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_goTypes, + DependencyIndexes: file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_depIdxs, + MessageInfos: file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_msgTypes, + }.Build() + File_temporal_server_chasm_lib_stream_proto_v1_request_response_proto = out.File + file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_goTypes = nil + file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_depIdxs = nil +} diff --git a/chasm/lib/stream/gen/streampb/v1/service.pb.go b/chasm/lib/stream/gen/streampb/v1/service.pb.go new file mode 100644 index 00000000000..e0bb7981366 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/service.pb.go @@ -0,0 +1,140 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/stream/proto/v1/service.proto + +package streampb + +import ( + reflect "reflect" + unsafe "unsafe" + + _ "go.temporal.io/server/api/common/v1" + _ "go.temporal.io/server/api/routing/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_temporal_server_chasm_lib_stream_proto_v1_service_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_stream_proto_v1_service_proto_rawDesc = "" + + "\n" + + "7temporal/server/chasm/lib/stream/proto/v1/service.proto\x12)temporal.server.chasm.lib.stream.proto.v1\x1a@temporal/server/chasm/lib/stream/proto/v1/request_response.proto\x1a0temporal/server/api/common/v1/api_category.proto\x1a.temporal/server/api/routing/v1/extension.proto2\xf8\x16\n" + + "\rStreamService\x12\xb7\x01\n" + + "\fCreateStream\x12>.temporal.server.chasm.lib.stream.proto.v1.CreateStreamRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.CreateStreamResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\xb4\x01\n" + + "\vAddMessages\x12=.temporal.server.chasm.lib.stream.proto.v1.AddMessagesRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.AddMessagesResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\xba\x01\n" + + "\rFinishWriting\x12?.temporal.server.chasm.lib.stream.proto.v1.FinishWritingRequest\x1a@.temporal.server.chasm.lib.stream.proto.v1.FinishWritingResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\xc8\x01\n" + + "\x11SubscribeWorkflow\x12C.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowRequest\x1aD.temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.workflow_id\x12\xb7\x01\n" + + "\fPollMessages\x12>.temporal.server.chasm.lib.stream.proto.v1.PollMessagesRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.PollMessagesResponse\"&\x8a\xb5\x18\x02\b\x02\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\xbd\x01\n" + + "\x0eDescribeStream\x12@.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamRequest\x1aA.temporal.server.chasm.lib.stream.proto.v1.DescribeStreamResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\xd1\x01\n" + + "\x14PollWorkflowMessages\x12F.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesRequest\x1aG.temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesResponse\"(\x8a\xb5\x18\x02\b\x02\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.workflow_id\x12\xd7\x01\n" + + "\x16DescribeWorkflowStream\x12H.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamRequest\x1aI.temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.workflow_id\x12\xce\x01\n" + + "\x13AddWorkflowMessages\x12E.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesRequest\x1aF.temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.workflow_id\x12\xd5\x01\n" + + "\x16RegisterStreamConsumer\x12H.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerRequest\x1aI.temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\xce\x01\n" + + "\x13AdvanceConsumerHead\x12E.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadRequest\x1aF.temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.workflow_id\x12\xb4\x01\n" + + "\vCloseStream\x12=.temporal.server.chasm.lib.stream.proto.v1.CloseStreamRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.CloseStreamResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\xbd\x01\n" + + "\x0eTruncateStream\x12@.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamRequest\x1aA.temporal.server.chasm.lib.stream.proto.v1.TruncateStreamResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_id\x12\x9a\x01\n" + + "\vListStreams\x12=.temporal.server.chasm.lib.stream.proto.v1.ListStreamsRequest\x1a>.temporal.server.chasm.lib.stream.proto.v1.ListStreamsResponse\"\f\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x02\b\x01\x12\xb7\x01\n" + + "\fDeleteStream\x12>.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamRequest\x1a?.temporal.server.chasm.lib.stream.proto.v1.DeleteStreamResponse\"&\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1c\x1a\x1afrontend_request.stream_idB>Z temporal.server.chasm.lib.stream.proto.v1.CreateStreamRequest + 1, // 1: temporal.server.chasm.lib.stream.proto.v1.StreamService.AddMessages:input_type -> temporal.server.chasm.lib.stream.proto.v1.AddMessagesRequest + 2, // 2: temporal.server.chasm.lib.stream.proto.v1.StreamService.FinishWriting:input_type -> temporal.server.chasm.lib.stream.proto.v1.FinishWritingRequest + 3, // 3: temporal.server.chasm.lib.stream.proto.v1.StreamService.SubscribeWorkflow:input_type -> temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowRequest + 4, // 4: temporal.server.chasm.lib.stream.proto.v1.StreamService.PollMessages:input_type -> temporal.server.chasm.lib.stream.proto.v1.PollMessagesRequest + 5, // 5: temporal.server.chasm.lib.stream.proto.v1.StreamService.DescribeStream:input_type -> temporal.server.chasm.lib.stream.proto.v1.DescribeStreamRequest + 6, // 6: temporal.server.chasm.lib.stream.proto.v1.StreamService.PollWorkflowMessages:input_type -> temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesRequest + 7, // 7: temporal.server.chasm.lib.stream.proto.v1.StreamService.DescribeWorkflowStream:input_type -> temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamRequest + 8, // 8: temporal.server.chasm.lib.stream.proto.v1.StreamService.AddWorkflowMessages:input_type -> temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesRequest + 9, // 9: temporal.server.chasm.lib.stream.proto.v1.StreamService.RegisterStreamConsumer:input_type -> temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerRequest + 10, // 10: temporal.server.chasm.lib.stream.proto.v1.StreamService.AdvanceConsumerHead:input_type -> temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadRequest + 11, // 11: temporal.server.chasm.lib.stream.proto.v1.StreamService.CloseStream:input_type -> temporal.server.chasm.lib.stream.proto.v1.CloseStreamRequest + 12, // 12: temporal.server.chasm.lib.stream.proto.v1.StreamService.TruncateStream:input_type -> temporal.server.chasm.lib.stream.proto.v1.TruncateStreamRequest + 13, // 13: temporal.server.chasm.lib.stream.proto.v1.StreamService.ListStreams:input_type -> temporal.server.chasm.lib.stream.proto.v1.ListStreamsRequest + 14, // 14: temporal.server.chasm.lib.stream.proto.v1.StreamService.DeleteStream:input_type -> temporal.server.chasm.lib.stream.proto.v1.DeleteStreamRequest + 15, // 15: temporal.server.chasm.lib.stream.proto.v1.StreamService.CreateStream:output_type -> temporal.server.chasm.lib.stream.proto.v1.CreateStreamResponse + 16, // 16: temporal.server.chasm.lib.stream.proto.v1.StreamService.AddMessages:output_type -> temporal.server.chasm.lib.stream.proto.v1.AddMessagesResponse + 17, // 17: temporal.server.chasm.lib.stream.proto.v1.StreamService.FinishWriting:output_type -> temporal.server.chasm.lib.stream.proto.v1.FinishWritingResponse + 18, // 18: temporal.server.chasm.lib.stream.proto.v1.StreamService.SubscribeWorkflow:output_type -> temporal.server.chasm.lib.stream.proto.v1.SubscribeWorkflowResponse + 19, // 19: temporal.server.chasm.lib.stream.proto.v1.StreamService.PollMessages:output_type -> temporal.server.chasm.lib.stream.proto.v1.PollMessagesResponse + 20, // 20: temporal.server.chasm.lib.stream.proto.v1.StreamService.DescribeStream:output_type -> temporal.server.chasm.lib.stream.proto.v1.DescribeStreamResponse + 21, // 21: temporal.server.chasm.lib.stream.proto.v1.StreamService.PollWorkflowMessages:output_type -> temporal.server.chasm.lib.stream.proto.v1.PollWorkflowMessagesResponse + 22, // 22: temporal.server.chasm.lib.stream.proto.v1.StreamService.DescribeWorkflowStream:output_type -> temporal.server.chasm.lib.stream.proto.v1.DescribeWorkflowStreamResponse + 23, // 23: temporal.server.chasm.lib.stream.proto.v1.StreamService.AddWorkflowMessages:output_type -> temporal.server.chasm.lib.stream.proto.v1.AddWorkflowMessagesResponse + 24, // 24: temporal.server.chasm.lib.stream.proto.v1.StreamService.RegisterStreamConsumer:output_type -> temporal.server.chasm.lib.stream.proto.v1.RegisterStreamConsumerResponse + 25, // 25: temporal.server.chasm.lib.stream.proto.v1.StreamService.AdvanceConsumerHead:output_type -> temporal.server.chasm.lib.stream.proto.v1.AdvanceConsumerHeadResponse + 26, // 26: temporal.server.chasm.lib.stream.proto.v1.StreamService.CloseStream:output_type -> temporal.server.chasm.lib.stream.proto.v1.CloseStreamResponse + 27, // 27: temporal.server.chasm.lib.stream.proto.v1.StreamService.TruncateStream:output_type -> temporal.server.chasm.lib.stream.proto.v1.TruncateStreamResponse + 28, // 28: temporal.server.chasm.lib.stream.proto.v1.StreamService.ListStreams:output_type -> temporal.server.chasm.lib.stream.proto.v1.ListStreamsResponse + 29, // 29: temporal.server.chasm.lib.stream.proto.v1.StreamService.DeleteStream:output_type -> temporal.server.chasm.lib.stream.proto.v1.DeleteStreamResponse + 15, // [15:30] is the sub-list for method output_type + 0, // [0:15] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_temporal_server_chasm_lib_stream_proto_v1_service_proto_init() } +func file_temporal_server_chasm_lib_stream_proto_v1_service_proto_init() { + if File_temporal_server_chasm_lib_stream_proto_v1_service_proto != nil { + return + } + file_temporal_server_chasm_lib_stream_proto_v1_request_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_stream_proto_v1_service_proto_rawDesc), len(file_temporal_server_chasm_lib_stream_proto_v1_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_temporal_server_chasm_lib_stream_proto_v1_service_proto_goTypes, + DependencyIndexes: file_temporal_server_chasm_lib_stream_proto_v1_service_proto_depIdxs, + }.Build() + File_temporal_server_chasm_lib_stream_proto_v1_service_proto = out.File + file_temporal_server_chasm_lib_stream_proto_v1_service_proto_goTypes = nil + file_temporal_server_chasm_lib_stream_proto_v1_service_proto_depIdxs = nil +} diff --git a/chasm/lib/stream/gen/streampb/v1/service_client.pb.go b/chasm/lib/stream/gen/streampb/v1/service_client.pb.go new file mode 100644 index 00000000000..946c94e2831 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/service_client.pb.go @@ -0,0 +1,713 @@ +// Code generated by protoc-gen-go-chasm. DO NOT EDIT. +package streampb + +import ( + "context" + "math/rand" + "time" + + "go.temporal.io/server/client/history" + "go.temporal.io/server/common" + "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/config" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/membership" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/primitives" + "go.uber.org/fx" + "google.golang.org/grpc" +) + +// StreamServiceLayeredClient is a client for StreamService. +type StreamServiceLayeredClient struct { + metricsHandler metrics.Handler + numShards int32 + redirector history.Redirector[StreamServiceClient] + retryPolicy backoff.RetryPolicy +} + +// NewStreamServiceLayeredClient initializes a new StreamServiceLayeredClient. +func NewStreamServiceLayeredClient( + lc fx.Lifecycle, + dc *dynamicconfig.Collection, + rpcFactory common.RPCFactory, + monitor membership.Monitor, + config *config.Persistence, + logger log.Logger, + metricsHandler metrics.Handler, +) (StreamServiceClient, error) { + resolver, err := monitor.GetResolver(primitives.HistoryService) + if err != nil { + return nil, err + } + connections := history.NewConnectionPool(resolver, rpcFactory, NewStreamServiceClient, logger, dynamicconfig.HistoryConnectionCloseDelay.Get(dc)) + var redirector history.Redirector[StreamServiceClient] + if dynamicconfig.HistoryClientOwnershipCachingEnabled.Get(dc)() { + redirector = history.NewCachingRedirector( + connections, + resolver, + logger, + dynamicconfig.HistoryClientOwnershipCachingStaleTTL.Get(dc), + ) + } else { + redirector = history.NewBasicRedirector(connections, resolver) + } + client := &StreamServiceLayeredClient{ + metricsHandler: metricsHandler, + redirector: redirector, + numShards: config.NumHistoryShards, + retryPolicy: common.CreateHistoryClientRetryPolicy(dynamicconfig.RetryUnboundedOnSystemResourceExhausted.Get(dc)), + } + lc.Append(fx.StopHook(client.Stop)) + return client, nil +} +func (c *StreamServiceLayeredClient) Stop() { + c.redirector.Close() +} +func (c *StreamServiceLayeredClient) callCreateStreamNoRetry( + ctx context.Context, + request *CreateStreamRequest, + opts ...grpc.CallOption, +) (*CreateStreamResponse, error) { + var response *CreateStreamResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.CreateStream"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.CreateStream(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) CreateStream( + ctx context.Context, + request *CreateStreamRequest, + opts ...grpc.CallOption, +) (*CreateStreamResponse, error) { + call := func(ctx context.Context) (*CreateStreamResponse, error) { + return c.callCreateStreamNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callAddMessagesNoRetry( + ctx context.Context, + request *AddMessagesRequest, + opts ...grpc.CallOption, +) (*AddMessagesResponse, error) { + var response *AddMessagesResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.AddMessages"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.AddMessages(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) AddMessages( + ctx context.Context, + request *AddMessagesRequest, + opts ...grpc.CallOption, +) (*AddMessagesResponse, error) { + call := func(ctx context.Context) (*AddMessagesResponse, error) { + return c.callAddMessagesNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callFinishWritingNoRetry( + ctx context.Context, + request *FinishWritingRequest, + opts ...grpc.CallOption, +) (*FinishWritingResponse, error) { + var response *FinishWritingResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.FinishWriting"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.FinishWriting(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) FinishWriting( + ctx context.Context, + request *FinishWritingRequest, + opts ...grpc.CallOption, +) (*FinishWritingResponse, error) { + call := func(ctx context.Context) (*FinishWritingResponse, error) { + return c.callFinishWritingNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callSubscribeWorkflowNoRetry( + ctx context.Context, + request *SubscribeWorkflowRequest, + opts ...grpc.CallOption, +) (*SubscribeWorkflowResponse, error) { + var response *SubscribeWorkflowResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.SubscribeWorkflow"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetWorkflowId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.SubscribeWorkflow(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) SubscribeWorkflow( + ctx context.Context, + request *SubscribeWorkflowRequest, + opts ...grpc.CallOption, +) (*SubscribeWorkflowResponse, error) { + call := func(ctx context.Context) (*SubscribeWorkflowResponse, error) { + return c.callSubscribeWorkflowNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callPollMessagesNoRetry( + ctx context.Context, + request *PollMessagesRequest, + opts ...grpc.CallOption, +) (*PollMessagesResponse, error) { + var response *PollMessagesResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.PollMessages"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.PollMessages(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) PollMessages( + ctx context.Context, + request *PollMessagesRequest, + opts ...grpc.CallOption, +) (*PollMessagesResponse, error) { + call := func(ctx context.Context) (*PollMessagesResponse, error) { + return c.callPollMessagesNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callDescribeStreamNoRetry( + ctx context.Context, + request *DescribeStreamRequest, + opts ...grpc.CallOption, +) (*DescribeStreamResponse, error) { + var response *DescribeStreamResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.DescribeStream"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.DescribeStream(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) DescribeStream( + ctx context.Context, + request *DescribeStreamRequest, + opts ...grpc.CallOption, +) (*DescribeStreamResponse, error) { + call := func(ctx context.Context) (*DescribeStreamResponse, error) { + return c.callDescribeStreamNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callPollWorkflowMessagesNoRetry( + ctx context.Context, + request *PollWorkflowMessagesRequest, + opts ...grpc.CallOption, +) (*PollWorkflowMessagesResponse, error) { + var response *PollWorkflowMessagesResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.PollWorkflowMessages"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetWorkflowId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.PollWorkflowMessages(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) PollWorkflowMessages( + ctx context.Context, + request *PollWorkflowMessagesRequest, + opts ...grpc.CallOption, +) (*PollWorkflowMessagesResponse, error) { + call := func(ctx context.Context) (*PollWorkflowMessagesResponse, error) { + return c.callPollWorkflowMessagesNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callDescribeWorkflowStreamNoRetry( + ctx context.Context, + request *DescribeWorkflowStreamRequest, + opts ...grpc.CallOption, +) (*DescribeWorkflowStreamResponse, error) { + var response *DescribeWorkflowStreamResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.DescribeWorkflowStream"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetWorkflowId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.DescribeWorkflowStream(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) DescribeWorkflowStream( + ctx context.Context, + request *DescribeWorkflowStreamRequest, + opts ...grpc.CallOption, +) (*DescribeWorkflowStreamResponse, error) { + call := func(ctx context.Context) (*DescribeWorkflowStreamResponse, error) { + return c.callDescribeWorkflowStreamNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callAddWorkflowMessagesNoRetry( + ctx context.Context, + request *AddWorkflowMessagesRequest, + opts ...grpc.CallOption, +) (*AddWorkflowMessagesResponse, error) { + var response *AddWorkflowMessagesResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.AddWorkflowMessages"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetWorkflowId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.AddWorkflowMessages(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) AddWorkflowMessages( + ctx context.Context, + request *AddWorkflowMessagesRequest, + opts ...grpc.CallOption, +) (*AddWorkflowMessagesResponse, error) { + call := func(ctx context.Context) (*AddWorkflowMessagesResponse, error) { + return c.callAddWorkflowMessagesNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callRegisterStreamConsumerNoRetry( + ctx context.Context, + request *RegisterStreamConsumerRequest, + opts ...grpc.CallOption, +) (*RegisterStreamConsumerResponse, error) { + var response *RegisterStreamConsumerResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.RegisterStreamConsumer"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.RegisterStreamConsumer(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) RegisterStreamConsumer( + ctx context.Context, + request *RegisterStreamConsumerRequest, + opts ...grpc.CallOption, +) (*RegisterStreamConsumerResponse, error) { + call := func(ctx context.Context) (*RegisterStreamConsumerResponse, error) { + return c.callRegisterStreamConsumerNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callAdvanceConsumerHeadNoRetry( + ctx context.Context, + request *AdvanceConsumerHeadRequest, + opts ...grpc.CallOption, +) (*AdvanceConsumerHeadResponse, error) { + var response *AdvanceConsumerHeadResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.AdvanceConsumerHead"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetWorkflowId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.AdvanceConsumerHead(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) AdvanceConsumerHead( + ctx context.Context, + request *AdvanceConsumerHeadRequest, + opts ...grpc.CallOption, +) (*AdvanceConsumerHeadResponse, error) { + call := func(ctx context.Context) (*AdvanceConsumerHeadResponse, error) { + return c.callAdvanceConsumerHeadNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callCloseStreamNoRetry( + ctx context.Context, + request *CloseStreamRequest, + opts ...grpc.CallOption, +) (*CloseStreamResponse, error) { + var response *CloseStreamResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.CloseStream"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.CloseStream(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) CloseStream( + ctx context.Context, + request *CloseStreamRequest, + opts ...grpc.CallOption, +) (*CloseStreamResponse, error) { + call := func(ctx context.Context) (*CloseStreamResponse, error) { + return c.callCloseStreamNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callTruncateStreamNoRetry( + ctx context.Context, + request *TruncateStreamRequest, + opts ...grpc.CallOption, +) (*TruncateStreamResponse, error) { + var response *TruncateStreamResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.TruncateStream"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.TruncateStream(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) TruncateStream( + ctx context.Context, + request *TruncateStreamRequest, + opts ...grpc.CallOption, +) (*TruncateStreamResponse, error) { + call := func(ctx context.Context) (*TruncateStreamResponse, error) { + return c.callTruncateStreamNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callListStreamsNoRetry( + ctx context.Context, + request *ListStreamsRequest, + opts ...grpc.CallOption, +) (*ListStreamsResponse, error) { + var response *ListStreamsResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.ListStreams"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := int32(rand.Intn(int(c.numShards)) + 1) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.ListStreams(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) ListStreams( + ctx context.Context, + request *ListStreamsRequest, + opts ...grpc.CallOption, +) (*ListStreamsResponse, error) { + call := func(ctx context.Context) (*ListStreamsResponse, error) { + return c.callListStreamsNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *StreamServiceLayeredClient) callDeleteStreamNoRetry( + ctx context.Context, + request *DeleteStreamRequest, + opts ...grpc.CallOption, +) (*DeleteStreamResponse, error) { + var response *DeleteStreamResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("StreamService.DeleteStream"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetStreamId(), c.numShards) + op := func(ctx context.Context, client StreamServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.DeleteStream(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *StreamServiceLayeredClient) DeleteStream( + ctx context.Context, + request *DeleteStreamRequest, + opts ...grpc.CallOption, +) (*DeleteStreamResponse, error) { + call := func(ctx context.Context) (*DeleteStreamResponse, error) { + return c.callDeleteStreamNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} diff --git a/chasm/lib/stream/gen/streampb/v1/service_grpc.pb.go b/chasm/lib/stream/gen/streampb/v1/service_grpc.pb.go new file mode 100644 index 00000000000..0a9d2f93a28 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/service_grpc.pb.go @@ -0,0 +1,640 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// plugins: +// - protoc-gen-go-grpc +// - protoc +// source: temporal/server/chasm/lib/stream/proto/v1/service.proto + +package streampb + +import ( + context "context" + + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + StreamService_CreateStream_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/CreateStream" + StreamService_AddMessages_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/AddMessages" + StreamService_FinishWriting_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/FinishWriting" + StreamService_SubscribeWorkflow_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/SubscribeWorkflow" + StreamService_PollMessages_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/PollMessages" + StreamService_DescribeStream_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/DescribeStream" + StreamService_PollWorkflowMessages_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/PollWorkflowMessages" + StreamService_DescribeWorkflowStream_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/DescribeWorkflowStream" + StreamService_AddWorkflowMessages_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/AddWorkflowMessages" + StreamService_RegisterStreamConsumer_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/RegisterStreamConsumer" + StreamService_AdvanceConsumerHead_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/AdvanceConsumerHead" + StreamService_CloseStream_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/CloseStream" + StreamService_TruncateStream_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/TruncateStream" + StreamService_ListStreams_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/ListStreams" + StreamService_DeleteStream_FullMethodName = "/temporal.server.chasm.lib.stream.proto.v1.StreamService/DeleteStream" +) + +// StreamServiceClient is the client API for StreamService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type StreamServiceClient interface { + CreateStream(ctx context.Context, in *CreateStreamRequest, opts ...grpc.CallOption) (*CreateStreamResponse, error) + AddMessages(ctx context.Context, in *AddMessagesRequest, opts ...grpc.CallOption) (*AddMessagesResponse, error) + FinishWriting(ctx context.Context, in *FinishWritingRequest, opts ...grpc.CallOption) (*FinishWritingResponse, error) + SubscribeWorkflow(ctx context.Context, in *SubscribeWorkflowRequest, opts ...grpc.CallOption) (*SubscribeWorkflowResponse, error) + PollMessages(ctx context.Context, in *PollMessagesRequest, opts ...grpc.CallOption) (*PollMessagesResponse, error) + DescribeStream(ctx context.Context, in *DescribeStreamRequest, opts ...grpc.CallOption) (*DescribeStreamResponse, error) + // Routed on the owner, because the stream it reads has no id of its own. + PollWorkflowMessages(ctx context.Context, in *PollWorkflowMessagesRequest, opts ...grpc.CallOption) (*PollWorkflowMessagesResponse, error) + DescribeWorkflowStream(ctx context.Context, in *DescribeWorkflowStreamRequest, opts ...grpc.CallOption) (*DescribeWorkflowStreamResponse, error) + AddWorkflowMessages(ctx context.Context, in *AddWorkflowMessagesRequest, opts ...grpc.CallOption) (*AddWorkflowMessagesResponse, error) + // Internal. History calls these on itself to reach a shard it does not own, + // which is the only way a step that spans two executions can work on a + // cluster with more than one history host. + RegisterStreamConsumer(ctx context.Context, in *RegisterStreamConsumerRequest, opts ...grpc.CallOption) (*RegisterStreamConsumerResponse, error) + AdvanceConsumerHead(ctx context.Context, in *AdvanceConsumerHeadRequest, opts ...grpc.CallOption) (*AdvanceConsumerHeadResponse, error) + CloseStream(ctx context.Context, in *CloseStreamRequest, opts ...grpc.CallOption) (*CloseStreamResponse, error) + TruncateStream(ctx context.Context, in *TruncateStreamRequest, opts ...grpc.CallOption) (*TruncateStreamResponse, error) + // Served on the frontend only: it queries visibility rather than a stream, + // so there is no business ID to route on and nothing for a shard to answer. + ListStreams(ctx context.Context, in *ListStreamsRequest, opts ...grpc.CallOption) (*ListStreamsResponse, error) + DeleteStream(ctx context.Context, in *DeleteStreamRequest, opts ...grpc.CallOption) (*DeleteStreamResponse, error) +} + +type streamServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewStreamServiceClient(cc grpc.ClientConnInterface) StreamServiceClient { + return &streamServiceClient{cc} +} + +func (c *streamServiceClient) CreateStream(ctx context.Context, in *CreateStreamRequest, opts ...grpc.CallOption) (*CreateStreamResponse, error) { + out := new(CreateStreamResponse) + err := c.cc.Invoke(ctx, StreamService_CreateStream_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) AddMessages(ctx context.Context, in *AddMessagesRequest, opts ...grpc.CallOption) (*AddMessagesResponse, error) { + out := new(AddMessagesResponse) + err := c.cc.Invoke(ctx, StreamService_AddMessages_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) FinishWriting(ctx context.Context, in *FinishWritingRequest, opts ...grpc.CallOption) (*FinishWritingResponse, error) { + out := new(FinishWritingResponse) + err := c.cc.Invoke(ctx, StreamService_FinishWriting_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) SubscribeWorkflow(ctx context.Context, in *SubscribeWorkflowRequest, opts ...grpc.CallOption) (*SubscribeWorkflowResponse, error) { + out := new(SubscribeWorkflowResponse) + err := c.cc.Invoke(ctx, StreamService_SubscribeWorkflow_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) PollMessages(ctx context.Context, in *PollMessagesRequest, opts ...grpc.CallOption) (*PollMessagesResponse, error) { + out := new(PollMessagesResponse) + err := c.cc.Invoke(ctx, StreamService_PollMessages_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) DescribeStream(ctx context.Context, in *DescribeStreamRequest, opts ...grpc.CallOption) (*DescribeStreamResponse, error) { + out := new(DescribeStreamResponse) + err := c.cc.Invoke(ctx, StreamService_DescribeStream_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) PollWorkflowMessages(ctx context.Context, in *PollWorkflowMessagesRequest, opts ...grpc.CallOption) (*PollWorkflowMessagesResponse, error) { + out := new(PollWorkflowMessagesResponse) + err := c.cc.Invoke(ctx, StreamService_PollWorkflowMessages_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) DescribeWorkflowStream(ctx context.Context, in *DescribeWorkflowStreamRequest, opts ...grpc.CallOption) (*DescribeWorkflowStreamResponse, error) { + out := new(DescribeWorkflowStreamResponse) + err := c.cc.Invoke(ctx, StreamService_DescribeWorkflowStream_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) AddWorkflowMessages(ctx context.Context, in *AddWorkflowMessagesRequest, opts ...grpc.CallOption) (*AddWorkflowMessagesResponse, error) { + out := new(AddWorkflowMessagesResponse) + err := c.cc.Invoke(ctx, StreamService_AddWorkflowMessages_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) RegisterStreamConsumer(ctx context.Context, in *RegisterStreamConsumerRequest, opts ...grpc.CallOption) (*RegisterStreamConsumerResponse, error) { + out := new(RegisterStreamConsumerResponse) + err := c.cc.Invoke(ctx, StreamService_RegisterStreamConsumer_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) AdvanceConsumerHead(ctx context.Context, in *AdvanceConsumerHeadRequest, opts ...grpc.CallOption) (*AdvanceConsumerHeadResponse, error) { + out := new(AdvanceConsumerHeadResponse) + err := c.cc.Invoke(ctx, StreamService_AdvanceConsumerHead_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) CloseStream(ctx context.Context, in *CloseStreamRequest, opts ...grpc.CallOption) (*CloseStreamResponse, error) { + out := new(CloseStreamResponse) + err := c.cc.Invoke(ctx, StreamService_CloseStream_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) TruncateStream(ctx context.Context, in *TruncateStreamRequest, opts ...grpc.CallOption) (*TruncateStreamResponse, error) { + out := new(TruncateStreamResponse) + err := c.cc.Invoke(ctx, StreamService_TruncateStream_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) ListStreams(ctx context.Context, in *ListStreamsRequest, opts ...grpc.CallOption) (*ListStreamsResponse, error) { + out := new(ListStreamsResponse) + err := c.cc.Invoke(ctx, StreamService_ListStreams_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *streamServiceClient) DeleteStream(ctx context.Context, in *DeleteStreamRequest, opts ...grpc.CallOption) (*DeleteStreamResponse, error) { + out := new(DeleteStreamResponse) + err := c.cc.Invoke(ctx, StreamService_DeleteStream_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// StreamServiceServer is the server API for StreamService service. +// All implementations must embed UnimplementedStreamServiceServer +// for forward compatibility +type StreamServiceServer interface { + CreateStream(context.Context, *CreateStreamRequest) (*CreateStreamResponse, error) + AddMessages(context.Context, *AddMessagesRequest) (*AddMessagesResponse, error) + FinishWriting(context.Context, *FinishWritingRequest) (*FinishWritingResponse, error) + SubscribeWorkflow(context.Context, *SubscribeWorkflowRequest) (*SubscribeWorkflowResponse, error) + PollMessages(context.Context, *PollMessagesRequest) (*PollMessagesResponse, error) + DescribeStream(context.Context, *DescribeStreamRequest) (*DescribeStreamResponse, error) + // Routed on the owner, because the stream it reads has no id of its own. + PollWorkflowMessages(context.Context, *PollWorkflowMessagesRequest) (*PollWorkflowMessagesResponse, error) + DescribeWorkflowStream(context.Context, *DescribeWorkflowStreamRequest) (*DescribeWorkflowStreamResponse, error) + AddWorkflowMessages(context.Context, *AddWorkflowMessagesRequest) (*AddWorkflowMessagesResponse, error) + // Internal. History calls these on itself to reach a shard it does not own, + // which is the only way a step that spans two executions can work on a + // cluster with more than one history host. + RegisterStreamConsumer(context.Context, *RegisterStreamConsumerRequest) (*RegisterStreamConsumerResponse, error) + AdvanceConsumerHead(context.Context, *AdvanceConsumerHeadRequest) (*AdvanceConsumerHeadResponse, error) + CloseStream(context.Context, *CloseStreamRequest) (*CloseStreamResponse, error) + TruncateStream(context.Context, *TruncateStreamRequest) (*TruncateStreamResponse, error) + // Served on the frontend only: it queries visibility rather than a stream, + // so there is no business ID to route on and nothing for a shard to answer. + ListStreams(context.Context, *ListStreamsRequest) (*ListStreamsResponse, error) + DeleteStream(context.Context, *DeleteStreamRequest) (*DeleteStreamResponse, error) + mustEmbedUnimplementedStreamServiceServer() +} + +// UnimplementedStreamServiceServer must be embedded to have forward compatible implementations. +type UnimplementedStreamServiceServer struct { +} + +func (UnimplementedStreamServiceServer) CreateStream(context.Context, *CreateStreamRequest) (*CreateStreamResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateStream not implemented") +} +func (UnimplementedStreamServiceServer) AddMessages(context.Context, *AddMessagesRequest) (*AddMessagesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AddMessages not implemented") +} +func (UnimplementedStreamServiceServer) FinishWriting(context.Context, *FinishWritingRequest) (*FinishWritingResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method FinishWriting not implemented") +} +func (UnimplementedStreamServiceServer) SubscribeWorkflow(context.Context, *SubscribeWorkflowRequest) (*SubscribeWorkflowResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubscribeWorkflow not implemented") +} +func (UnimplementedStreamServiceServer) PollMessages(context.Context, *PollMessagesRequest) (*PollMessagesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PollMessages not implemented") +} +func (UnimplementedStreamServiceServer) DescribeStream(context.Context, *DescribeStreamRequest) (*DescribeStreamResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DescribeStream not implemented") +} +func (UnimplementedStreamServiceServer) PollWorkflowMessages(context.Context, *PollWorkflowMessagesRequest) (*PollWorkflowMessagesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PollWorkflowMessages not implemented") +} +func (UnimplementedStreamServiceServer) DescribeWorkflowStream(context.Context, *DescribeWorkflowStreamRequest) (*DescribeWorkflowStreamResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DescribeWorkflowStream not implemented") +} +func (UnimplementedStreamServiceServer) AddWorkflowMessages(context.Context, *AddWorkflowMessagesRequest) (*AddWorkflowMessagesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AddWorkflowMessages not implemented") +} +func (UnimplementedStreamServiceServer) RegisterStreamConsumer(context.Context, *RegisterStreamConsumerRequest) (*RegisterStreamConsumerResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RegisterStreamConsumer not implemented") +} +func (UnimplementedStreamServiceServer) AdvanceConsumerHead(context.Context, *AdvanceConsumerHeadRequest) (*AdvanceConsumerHeadResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AdvanceConsumerHead not implemented") +} +func (UnimplementedStreamServiceServer) CloseStream(context.Context, *CloseStreamRequest) (*CloseStreamResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CloseStream not implemented") +} +func (UnimplementedStreamServiceServer) TruncateStream(context.Context, *TruncateStreamRequest) (*TruncateStreamResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method TruncateStream not implemented") +} +func (UnimplementedStreamServiceServer) ListStreams(context.Context, *ListStreamsRequest) (*ListStreamsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListStreams not implemented") +} +func (UnimplementedStreamServiceServer) DeleteStream(context.Context, *DeleteStreamRequest) (*DeleteStreamResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteStream not implemented") +} +func (UnimplementedStreamServiceServer) mustEmbedUnimplementedStreamServiceServer() {} + +// UnsafeStreamServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to StreamServiceServer will +// result in compilation errors. +type UnsafeStreamServiceServer interface { + mustEmbedUnimplementedStreamServiceServer() +} + +func RegisterStreamServiceServer(s grpc.ServiceRegistrar, srv StreamServiceServer) { + s.RegisterService(&StreamService_ServiceDesc, srv) +} + +func _StreamService_CreateStream_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateStreamRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).CreateStream(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_CreateStream_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).CreateStream(ctx, req.(*CreateStreamRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_AddMessages_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AddMessagesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).AddMessages(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_AddMessages_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).AddMessages(ctx, req.(*AddMessagesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_FinishWriting_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FinishWritingRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).FinishWriting(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_FinishWriting_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).FinishWriting(ctx, req.(*FinishWritingRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_SubscribeWorkflow_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SubscribeWorkflowRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).SubscribeWorkflow(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_SubscribeWorkflow_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).SubscribeWorkflow(ctx, req.(*SubscribeWorkflowRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_PollMessages_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PollMessagesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).PollMessages(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_PollMessages_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).PollMessages(ctx, req.(*PollMessagesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_DescribeStream_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DescribeStreamRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).DescribeStream(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_DescribeStream_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).DescribeStream(ctx, req.(*DescribeStreamRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_PollWorkflowMessages_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PollWorkflowMessagesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).PollWorkflowMessages(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_PollWorkflowMessages_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).PollWorkflowMessages(ctx, req.(*PollWorkflowMessagesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_DescribeWorkflowStream_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DescribeWorkflowStreamRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).DescribeWorkflowStream(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_DescribeWorkflowStream_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).DescribeWorkflowStream(ctx, req.(*DescribeWorkflowStreamRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_AddWorkflowMessages_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AddWorkflowMessagesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).AddWorkflowMessages(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_AddWorkflowMessages_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).AddWorkflowMessages(ctx, req.(*AddWorkflowMessagesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_RegisterStreamConsumer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RegisterStreamConsumerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).RegisterStreamConsumer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_RegisterStreamConsumer_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).RegisterStreamConsumer(ctx, req.(*RegisterStreamConsumerRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_AdvanceConsumerHead_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AdvanceConsumerHeadRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).AdvanceConsumerHead(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_AdvanceConsumerHead_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).AdvanceConsumerHead(ctx, req.(*AdvanceConsumerHeadRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_CloseStream_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CloseStreamRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).CloseStream(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_CloseStream_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).CloseStream(ctx, req.(*CloseStreamRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_TruncateStream_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TruncateStreamRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).TruncateStream(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_TruncateStream_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).TruncateStream(ctx, req.(*TruncateStreamRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_ListStreams_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListStreamsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).ListStreams(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_ListStreams_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).ListStreams(ctx, req.(*ListStreamsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _StreamService_DeleteStream_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteStreamRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StreamServiceServer).DeleteStream(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StreamService_DeleteStream_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StreamServiceServer).DeleteStream(ctx, req.(*DeleteStreamRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// StreamService_ServiceDesc is the grpc.ServiceDesc for StreamService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var StreamService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "temporal.server.chasm.lib.stream.proto.v1.StreamService", + HandlerType: (*StreamServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CreateStream", + Handler: _StreamService_CreateStream_Handler, + }, + { + MethodName: "AddMessages", + Handler: _StreamService_AddMessages_Handler, + }, + { + MethodName: "FinishWriting", + Handler: _StreamService_FinishWriting_Handler, + }, + { + MethodName: "SubscribeWorkflow", + Handler: _StreamService_SubscribeWorkflow_Handler, + }, + { + MethodName: "PollMessages", + Handler: _StreamService_PollMessages_Handler, + }, + { + MethodName: "DescribeStream", + Handler: _StreamService_DescribeStream_Handler, + }, + { + MethodName: "PollWorkflowMessages", + Handler: _StreamService_PollWorkflowMessages_Handler, + }, + { + MethodName: "DescribeWorkflowStream", + Handler: _StreamService_DescribeWorkflowStream_Handler, + }, + { + MethodName: "AddWorkflowMessages", + Handler: _StreamService_AddWorkflowMessages_Handler, + }, + { + MethodName: "RegisterStreamConsumer", + Handler: _StreamService_RegisterStreamConsumer_Handler, + }, + { + MethodName: "AdvanceConsumerHead", + Handler: _StreamService_AdvanceConsumerHead_Handler, + }, + { + MethodName: "CloseStream", + Handler: _StreamService_CloseStream_Handler, + }, + { + MethodName: "TruncateStream", + Handler: _StreamService_TruncateStream_Handler, + }, + { + MethodName: "ListStreams", + Handler: _StreamService_ListStreams_Handler, + }, + { + MethodName: "DeleteStream", + Handler: _StreamService_DeleteStream_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "temporal/server/chasm/lib/stream/proto/v1/service.proto", +} diff --git a/chasm/lib/stream/gen/streampb/v1/stream_state.go-helpers.pb.go b/chasm/lib/stream/gen/streampb/v1/stream_state.go-helpers.pb.go new file mode 100644 index 00000000000..99af3b53701 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/stream_state.go-helpers.pb.go @@ -0,0 +1,191 @@ +// Code generated by protoc-gen-go-helpers. DO NOT EDIT. +package streampb + +import ( + "google.golang.org/protobuf/proto" +) + +// Marshal an object of type StreamState to the protobuf v3 wire format +func (val *StreamState) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StreamState from the protobuf v3 wire format +func (val *StreamState) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StreamState) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StreamState values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StreamState) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StreamState + switch t := that.(type) { + case *StreamState: + that1 = t + case StreamState: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type ProducerCursor to the protobuf v3 wire format +func (val *ProducerCursor) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type ProducerCursor from the protobuf v3 wire format +func (val *ProducerCursor) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *ProducerCursor) Size() int { + return proto.Size(val) +} + +// Equal returns whether two ProducerCursor values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *ProducerCursor) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *ProducerCursor + switch t := that.(type) { + case *ProducerCursor: + that1 = t + case ProducerCursor: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type ConsumerCursor to the protobuf v3 wire format +func (val *ConsumerCursor) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type ConsumerCursor from the protobuf v3 wire format +func (val *ConsumerCursor) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *ConsumerCursor) Size() int { + return proto.Size(val) +} + +// Equal returns whether two ConsumerCursor values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *ConsumerCursor) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *ConsumerCursor + switch t := that.(type) { + case *ConsumerCursor: + that1 = t + case ConsumerCursor: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type WorkflowStreamCursor to the protobuf v3 wire format +func (val *WorkflowStreamCursor) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type WorkflowStreamCursor from the protobuf v3 wire format +func (val *WorkflowStreamCursor) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *WorkflowStreamCursor) Size() int { + return proto.Size(val) +} + +// Equal returns whether two WorkflowStreamCursor values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *WorkflowStreamCursor) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *WorkflowStreamCursor + switch t := that.(type) { + case *WorkflowStreamCursor: + that1 = t + case WorkflowStreamCursor: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type StreamLifecycle to the protobuf v3 wire format +func (val *StreamLifecycle) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StreamLifecycle from the protobuf v3 wire format +func (val *StreamLifecycle) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StreamLifecycle) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StreamLifecycle values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StreamLifecycle) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StreamLifecycle + switch t := that.(type) { + case *StreamLifecycle: + that1 = t + case StreamLifecycle: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/chasm/lib/stream/gen/streampb/v1/stream_state.pb.go b/chasm/lib/stream/gen/streampb/v1/stream_state.pb.go new file mode 100644 index 00000000000..bec3fb2467d --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/stream_state.pb.go @@ -0,0 +1,665 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/stream/proto/v1/stream_state.proto + +package streampb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + v1 "go.temporal.io/api/common/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Size is O(producers + consumers), never O(messages). Payload bytes live in +// the log, not here, which is what keeps this off the CHASM partial-read path. +type StreamState struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Visibility frontier. Readers never observe an offset at or past this. + HeadOffset int64 `protobuf:"varint,1,opt,name=head_offset,json=headOffset,proto3" json:"head_offset,omitempty"` + // Truncation floor. Offsets below this are gone. + BaseOffset int64 `protobuf:"varint,2,opt,name=base_offset,json=baseOffset,proto3" json:"base_offset,omitempty"` // Chains log nodes so a stale node from an abandoned append is rejected on + Closed bool `protobuf:"varint,4,opt,name=closed,proto3" json:"closed,omitempty"` + CloseReason *v1.Payload `protobuf:"bytes,5,opt,name=close_reason,json=closeReason,proto3" json:"close_reason,omitempty"` + // Bumped on ownership change so a stale producer's write fails. + OwnerEpoch int64 `protobuf:"varint,6,opt,name=owner_epoch,json=ownerEpoch,proto3" json:"owner_epoch,omitempty"` + // Immutable once set. Offsets roll to a new log tree every bucket_size so no + // single storage partition grows with the stream. + BucketSize int64 `protobuf:"varint,7,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` + // Identity of the log this stream writes to. Bucket trees are derived from + // it, so there is no per-bucket index to store. + CollectionId string `protobuf:"bytes,8,opt,name=collection_id,json=collectionId,proto3" json:"collection_id,omitempty"` + Producers map[string]*ProducerCursor `protobuf:"bytes,9,rep,name=producers,proto3" json:"producers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Consumers map[string]*ConsumerCursor `protobuf:"bytes,10,rep,name=consumers,proto3" json:"consumers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Lifecycle *StreamLifecycle `protobuf:"bytes,11,opt,name=lifecycle,proto3" json:"lifecycle,omitempty"` + // Set when a successor run takes ownership, so an in-flight poll can follow + // the chain instead of stalling on a superseded run. + RedirectRunId string `protobuf:"bytes,12,opt,name=redirect_run_id,json=redirectRunId,proto3" json:"redirect_run_id,omitempty"` + // Wall-clock close time, used to schedule retention deletion. + CloseTime *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=close_time,json=closeTime,proto3" json:"close_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamState) Reset() { + *x = StreamState{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamState) ProtoMessage() {} + +func (x *StreamState) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamState.ProtoReflect.Descriptor instead. +func (*StreamState) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDescGZIP(), []int{0} +} + +func (x *StreamState) GetHeadOffset() int64 { + if x != nil { + return x.HeadOffset + } + return 0 +} + +func (x *StreamState) GetBaseOffset() int64 { + if x != nil { + return x.BaseOffset + } + return 0 +} + +func (x *StreamState) GetClosed() bool { + if x != nil { + return x.Closed + } + return false +} + +func (x *StreamState) GetCloseReason() *v1.Payload { + if x != nil { + return x.CloseReason + } + return nil +} + +func (x *StreamState) GetOwnerEpoch() int64 { + if x != nil { + return x.OwnerEpoch + } + return 0 +} + +func (x *StreamState) GetBucketSize() int64 { + if x != nil { + return x.BucketSize + } + return 0 +} + +func (x *StreamState) GetCollectionId() string { + if x != nil { + return x.CollectionId + } + return "" +} + +func (x *StreamState) GetProducers() map[string]*ProducerCursor { + if x != nil { + return x.Producers + } + return nil +} + +func (x *StreamState) GetConsumers() map[string]*ConsumerCursor { + if x != nil { + return x.Consumers + } + return nil +} + +func (x *StreamState) GetLifecycle() *StreamLifecycle { + if x != nil { + return x.Lifecycle + } + return nil +} + +func (x *StreamState) GetRedirectRunId() string { + if x != nil { + return x.RedirectRunId + } + return "" +} + +func (x *StreamState) GetCloseTime() *timestamppb.Timestamp { + if x != nil { + return x.CloseTime + } + return nil +} + +type ProducerCursor struct { + state protoimpl.MessageState `protogen:"open.v1"` + Seq int64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` + FirstOffset int64 `protobuf:"varint,2,opt,name=first_offset,json=firstOffset,proto3" json:"first_offset,omitempty"` + Count int64 `protobuf:"varint,3,opt,name=count,proto3" json:"count,omitempty"` + // Distinguishes a genuine retry from a client reusing a sequence with + // different content, which must be rejected rather than deduplicated. + ContentHash []byte `protobuf:"bytes,4,opt,name=content_hash,json=contentHash,proto3" json:"content_hash,omitempty"` + // Set by FinishWriting. Ends this producer's writes without closing the + // stream for anyone else. + Fenced bool `protobuf:"varint,5,opt,name=fenced,proto3" json:"fenced,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProducerCursor) Reset() { + *x = ProducerCursor{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProducerCursor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProducerCursor) ProtoMessage() {} + +func (x *ProducerCursor) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProducerCursor.ProtoReflect.Descriptor instead. +func (*ProducerCursor) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDescGZIP(), []int{1} +} + +func (x *ProducerCursor) GetSeq() int64 { + if x != nil { + return x.Seq + } + return 0 +} + +func (x *ProducerCursor) GetFirstOffset() int64 { + if x != nil { + return x.FirstOffset + } + return 0 +} + +func (x *ProducerCursor) GetCount() int64 { + if x != nil { + return x.Count + } + return 0 +} + +func (x *ProducerCursor) GetContentHash() []byte { + if x != nil { + return x.ContentHash + } + return nil +} + +func (x *ProducerCursor) GetFenced() bool { + if x != nil { + return x.Fenced + } + return false +} + +type ConsumerCursor struct { + state protoimpl.MessageState `protogen:"open.v1"` + WorkflowId string `protobuf:"bytes,1,opt,name=workflow_id,json=workflowId,proto3" json:"workflow_id,omitempty"` + RunId string `protobuf:"bytes,2,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + // How far this consumer has read. Used to decide whether it needs waking, + // not to decide what the stream may drop. + Offset int64 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"` + Active bool `protobuf:"varint,4,opt,name=active,proto3" json:"active,omitempty"` + // Set when the consumer is a workflow in another execution, which is the + // only case that has to be told the frontier moved. A workflow consuming a + // stream it owns sees that while closing its own transaction. + External bool `protobuf:"varint,5,opt,name=external,proto3" json:"external,omitempty"` + // The oldest offset this consumer's History still depends on. + // + // It is where the subscription started, not where it has read to. A range + // this consumer already consumed is recorded in its History and has to be + // re-readable for the workflow to replay, so bytes below the read position + // are exactly the ones a replay needs most. It stays put while the consumer + // is active and is released when it deregisters. + ReplayFloor int64 `protobuf:"varint,6,opt,name=replay_floor,json=replayFloor,proto3" json:"replay_floor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConsumerCursor) Reset() { + *x = ConsumerCursor{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConsumerCursor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConsumerCursor) ProtoMessage() {} + +func (x *ConsumerCursor) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConsumerCursor.ProtoReflect.Descriptor instead. +func (*ConsumerCursor) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDescGZIP(), []int{2} +} + +func (x *ConsumerCursor) GetWorkflowId() string { + if x != nil { + return x.WorkflowId + } + return "" +} + +func (x *ConsumerCursor) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *ConsumerCursor) GetOffset() int64 { + if x != nil { + return x.Offset + } + return 0 +} + +func (x *ConsumerCursor) GetActive() bool { + if x != nil { + return x.Active + } + return false +} + +func (x *ConsumerCursor) GetExternal() bool { + if x != nil { + return x.External + } + return false +} + +func (x *ConsumerCursor) GetReplayFloor() int64 { + if x != nil { + return x.ReplayFloor + } + return 0 +} + +// A consuming Workflow's position in a stream. This lives in the consuming +// Workflow's own state rather than on the stream, so advancing it commits in +// the same transaction as the WorkflowTaskCompleted event that records the +// range. Keeping it on the stream would make the advance a cross-execution +// write, and a crash between the two would either redeliver or skip. +type WorkflowStreamCursor struct { + state protoimpl.MessageState `protogen:"open.v1"` + StreamId string `protobuf:"bytes,1,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` + // Enough to address the log without reading the stream component first. + CollectionId string `protobuf:"bytes,2,opt,name=collection_id,json=collectionId,proto3" json:"collection_id,omitempty"` + BucketSize int64 `protobuf:"varint,3,opt,name=bucket_size,json=bucketSize,proto3" json:"bucket_size,omitempty"` + // Next offset to deliver. + Offset int64 `protobuf:"varint,4,opt,name=offset,proto3" json:"offset,omitempty"` + // The stream's frontier as of the last delivery. A workflow consuming a + // stream in another execution cannot read the real frontier while closing its + // own transaction, so this is what tells it a capped slice left more behind. + KnownHead int64 `protobuf:"varint,8,opt,name=known_head,json=knownHead,proto3" json:"known_head,omitempty"` + // Set when the stream lives in another execution, which decides whether the + // frontier is read locally or taken from known_head. + External bool `protobuf:"varint,9,opt,name=external,proto3" json:"external,omitempty"` + // The range attached to the Workflow Task currently in flight. Recorded on + // the event that closes that task, then folded into offset. An empty range + // is still recorded: a task where the subscription saw nothing is a fact + // replay has to reproduce. + PendingFrom int64 `protobuf:"varint,5,opt,name=pending_from,json=pendingFrom,proto3" json:"pending_from,omitempty"` + PendingTo int64 `protobuf:"varint,6,opt,name=pending_to,json=pendingTo,proto3" json:"pending_to,omitempty"` + HasPending bool `protobuf:"varint,7,opt,name=has_pending,json=hasPending,proto3" json:"has_pending,omitempty"` + // Where this subscription began reading. Retained separately from offset, + // which advances, because replay has to tell "committed nothing yet" apart + // from "the events recording what was committed are not in this page". + StartOffset int64 `protobuf:"varint,10,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkflowStreamCursor) Reset() { + *x = WorkflowStreamCursor{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkflowStreamCursor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkflowStreamCursor) ProtoMessage() {} + +func (x *WorkflowStreamCursor) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkflowStreamCursor.ProtoReflect.Descriptor instead. +func (*WorkflowStreamCursor) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDescGZIP(), []int{3} +} + +func (x *WorkflowStreamCursor) GetStreamId() string { + if x != nil { + return x.StreamId + } + return "" +} + +func (x *WorkflowStreamCursor) GetCollectionId() string { + if x != nil { + return x.CollectionId + } + return "" +} + +func (x *WorkflowStreamCursor) GetBucketSize() int64 { + if x != nil { + return x.BucketSize + } + return 0 +} + +func (x *WorkflowStreamCursor) GetOffset() int64 { + if x != nil { + return x.Offset + } + return 0 +} + +func (x *WorkflowStreamCursor) GetKnownHead() int64 { + if x != nil { + return x.KnownHead + } + return 0 +} + +func (x *WorkflowStreamCursor) GetExternal() bool { + if x != nil { + return x.External + } + return false +} + +func (x *WorkflowStreamCursor) GetPendingFrom() int64 { + if x != nil { + return x.PendingFrom + } + return 0 +} + +func (x *WorkflowStreamCursor) GetPendingTo() int64 { + if x != nil { + return x.PendingTo + } + return 0 +} + +func (x *WorkflowStreamCursor) GetHasPending() bool { + if x != nil { + return x.HasPending + } + return false +} + +func (x *WorkflowStreamCursor) GetStartOffset() int64 { + if x != nil { + return x.StartOffset + } + return 0 +} + +type StreamLifecycle struct { + state protoimpl.MessageState `protogen:"open.v1"` + // How long a closed stream stays readable before it is deleted. + Retention *durationpb.Duration `protobuf:"bytes,1,opt,name=retention,proto3" json:"retention,omitempty"` + // Cap on readable messages. Older whole buckets are reclaimed once the floor + // passes them, so a capped stream has bounded storage. + MaxItems int64 `protobuf:"varint,2,opt,name=max_items,json=maxItems,proto3" json:"max_items,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamLifecycle) Reset() { + *x = StreamLifecycle{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamLifecycle) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamLifecycle) ProtoMessage() {} + +func (x *StreamLifecycle) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamLifecycle.ProtoReflect.Descriptor instead. +func (*StreamLifecycle) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDescGZIP(), []int{4} +} + +func (x *StreamLifecycle) GetRetention() *durationpb.Duration { + if x != nil { + return x.Retention + } + return nil +} + +func (x *StreamLifecycle) GetMaxItems() int64 { + if x != nil { + return x.MaxItems + } + return 0 +} + +var File_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDesc = "" + + "\n" + + "Z temporal.api.common.v1.Payload + 5, // 1: temporal.server.chasm.lib.stream.proto.v1.StreamState.producers:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamState.ProducersEntry + 6, // 2: temporal.server.chasm.lib.stream.proto.v1.StreamState.consumers:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamState.ConsumersEntry + 4, // 3: temporal.server.chasm.lib.stream.proto.v1.StreamState.lifecycle:type_name -> temporal.server.chasm.lib.stream.proto.v1.StreamLifecycle + 8, // 4: temporal.server.chasm.lib.stream.proto.v1.StreamState.close_time:type_name -> google.protobuf.Timestamp + 9, // 5: temporal.server.chasm.lib.stream.proto.v1.StreamLifecycle.retention:type_name -> google.protobuf.Duration + 1, // 6: temporal.server.chasm.lib.stream.proto.v1.StreamState.ProducersEntry.value:type_name -> temporal.server.chasm.lib.stream.proto.v1.ProducerCursor + 2, // 7: temporal.server.chasm.lib.stream.proto.v1.StreamState.ConsumersEntry.value:type_name -> temporal.server.chasm.lib.stream.proto.v1.ConsumerCursor + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_init() } +func file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_init() { + if File_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDesc), len(file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_rawDesc)), + NumEnums: 0, + NumMessages: 7, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_goTypes, + DependencyIndexes: file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_depIdxs, + MessageInfos: file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_msgTypes, + }.Build() + File_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto = out.File + file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_goTypes = nil + file_temporal_server_chasm_lib_stream_proto_v1_stream_state_proto_depIdxs = nil +} diff --git a/chasm/lib/stream/gen/streampb/v1/tasks.go-helpers.pb.go b/chasm/lib/stream/gen/streampb/v1/tasks.go-helpers.pb.go new file mode 100644 index 00000000000..9644b7bbe02 --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/tasks.go-helpers.pb.go @@ -0,0 +1,80 @@ +// Code generated by protoc-gen-go-helpers. DO NOT EDIT. +package streampb + +import ( + "google.golang.org/protobuf/proto" +) + +// Marshal an object of type StreamRetentionTask to the protobuf v3 wire format +func (val *StreamRetentionTask) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StreamRetentionTask from the protobuf v3 wire format +func (val *StreamRetentionTask) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StreamRetentionTask) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StreamRetentionTask values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StreamRetentionTask) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StreamRetentionTask + switch t := that.(type) { + case *StreamRetentionTask: + that1 = t + case StreamRetentionTask: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type StreamNotifyConsumersTask to the protobuf v3 wire format +func (val *StreamNotifyConsumersTask) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StreamNotifyConsumersTask from the protobuf v3 wire format +func (val *StreamNotifyConsumersTask) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StreamNotifyConsumersTask) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StreamNotifyConsumersTask values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StreamNotifyConsumersTask) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StreamNotifyConsumersTask + switch t := that.(type) { + case *StreamNotifyConsumersTask: + that1 = t + case StreamNotifyConsumersTask: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/chasm/lib/stream/gen/streampb/v1/tasks.pb.go b/chasm/lib/stream/gen/streampb/v1/tasks.pb.go new file mode 100644 index 00000000000..2b944c7d24d --- /dev/null +++ b/chasm/lib/stream/gen/streampb/v1/tasks.pb.go @@ -0,0 +1,160 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/stream/proto/v1/tasks.proto + +package streampb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Fires at close_time plus retention. A closed stream stays readable until +// then, which is what removes the shutdown handshake the signal-based +// implementation forces on producers and consumers. +type StreamRetentionTask struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamRetentionTask) Reset() { + *x = StreamRetentionTask{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_tasks_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamRetentionTask) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamRetentionTask) ProtoMessage() {} + +func (x *StreamRetentionTask) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_tasks_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamRetentionTask.ProtoReflect.Descriptor instead. +func (*StreamRetentionTask) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_tasks_proto_rawDescGZIP(), []int{0} +} + +// Fires after an append that leaves a registered consumer behind. Appending +// never schedules a workflow task on its own, because a stream item is data an +// execution produced rather than a decision input to it. A workflow that +// subscribed is the exception: it asked to be told, and it cannot find out any +// other way, since nothing else it does would notice the stream moved. +type StreamNotifyConsumersTask struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamNotifyConsumersTask) Reset() { + *x = StreamNotifyConsumersTask{} + mi := &file_temporal_server_chasm_lib_stream_proto_v1_tasks_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamNotifyConsumersTask) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamNotifyConsumersTask) ProtoMessage() {} + +func (x *StreamNotifyConsumersTask) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_stream_proto_v1_tasks_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamNotifyConsumersTask.ProtoReflect.Descriptor instead. +func (*StreamNotifyConsumersTask) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_stream_proto_v1_tasks_proto_rawDescGZIP(), []int{1} +} + +var File_temporal_server_chasm_lib_stream_proto_v1_tasks_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_stream_proto_v1_tasks_proto_rawDesc = "" + + "\n" + + "5temporal/server/chasm/lib/stream/proto/v1/tasks.proto\x12)temporal.server.chasm.lib.stream.proto.v1\"\x15\n" + + "\x13StreamRetentionTask\"\x1b\n" + + "\x19StreamNotifyConsumersTaskB>Z= toOffset { + return blobs, startOffsets, nil + } + + pageSize := maxBatches + if pageSize <= 0 { + pageSize = defaultReadPageSize + } + + for bucket := BucketOf(fromOffset, bucketSize); BucketStart(bucket, bucketSize) < toOffset; bucket++ { + bucketStart := BucketStart(bucket, bucketSize) + bucketEnd := bucketStart + bucketSize + + resp, err := execMgr.ReadStreamLog(ctx, &persistence.InternalReadStreamLogRequest{ + ShardID: shardID, + NamespaceID: namespaceID, + CollectionID: collectionID, + Bucket: bucket, + MinOffset: max(fromOffset, bucketStart), + MaxOffset: min(toOffset, bucketEnd), + PageSize: pageSize, + }) + if err != nil { + return nil, nil, err + } + + blobs = append(blobs, resp.Batches...) + startOffsets = append(startOffsets, resp.StartOffsets...) + + if maxBatches > 0 && len(blobs) >= maxBatches { + return blobs[:maxBatches], startOffsets[:maxBatches], nil + } + } + return blobs, startOffsets, nil +} + +// DeleteBucket removes a whole bucket's tree. Reclaiming a truncated stream one +// bucket at a time is the point of bucketing: a partition is dropped outright +// rather than leaving a tombstone per message. +func DeleteBucket( + ctx context.Context, + execMgr persistence.ExecutionManager, + shardID int32, + namespaceID string, + collectionID string, + bucket int64, +) error { + return execMgr.DeleteStreamLogBucket(ctx, &persistence.InternalDeleteStreamLogBucketRequest{ + ShardID: shardID, + NamespaceID: namespaceID, + CollectionID: collectionID, + Bucket: bucket, + }) +} + +// ReclaimableBuckets lists buckets that lie entirely below the readable floor +// and can therefore be deleted. A bucket is only reclaimable once every offset +// it holds is unreadable, so this can never drop data a reader may still ask +// for. +func ReclaimableBuckets(previousBase, newBase, bucketSize int64) []int64 { + var out []int64 + for b := BucketOf(previousBase, bucketSize); BucketStart(b+1, bucketSize) <= newBase; b++ { + out = append(out, b) + } + return out +} diff --git a/chasm/lib/stream/messages.go b/chasm/lib/stream/messages.go new file mode 100644 index 00000000000..37fb96df504 --- /dev/null +++ b/chasm/lib/stream/messages.go @@ -0,0 +1,125 @@ +package stream + +import ( + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + streampb "go.temporal.io/api/stream/v1" + streamlib "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "google.golang.org/protobuf/proto" +) + +// CollectMessages decodes the batches covering a range and trims to the +// requested window. Decoding happens only here and only on the batches a read +// actually touches; the store never interprets them, and user payloads stay +// opaque because the codec runs in the SDK. +func CollectMessages( + blobs []*commonpb.DataBlob, + startOffsets []int64, + from int64, + head int64, + maxMessages int, + topics []string, +) ([]*streamlib.StreamMessage, int64, error) { + wanted := make(map[string]struct{}, len(topics)) + for _, t := range topics { + wanted[t] = struct{}{} + } + + var out []*streamlib.StreamMessage + next := from + for i, blob := range blobs { + var batch streamlib.StreamMessageBatch + if err := proto.Unmarshal(blob.GetData(), &batch); err != nil { + return nil, 0, err + } + for j, msg := range batch.GetMessages() { + offset := startOffsets[i] + int64(j) + if offset < from || offset >= head { + continue + } + if len(out) >= maxMessages { + return out, next, nil + } + next = offset + 1 + if len(wanted) > 0 { + if _, ok := wanted[msg.GetTopic()]; !ok { + continue + } + } + // Set here rather than stored: it is decided by where the + // message sits in the log, not by what the producer wrote. + msg.Offset = offset + out = append(out, msg) + } + } + return out, next, nil +} + +// ToAPIMessages converts stored messages to the shape carried on a Workflow +// Task. Control messages are dropped: they steer the log itself and mean +// nothing to a consumer. +func ToAPIMessages(in []*streamlib.StreamMessage) []*streampb.StreamMessage { + out := make([]*streampb.StreamMessage, 0, len(in)) + for _, m := range in { + if m.GetKind() != streamlib.STREAM_MESSAGE_KIND_DATA { + continue + } + out = append(out, &streampb.StreamMessage{ + Body: m.GetBody(), + Metadata: m.GetMetadata(), + Topic: m.GetTopic(), + TopicSequence: m.GetTopicSequence(), + }) + } + return out +} + +// CapByBytes trims a contiguous run of messages to a byte budget and returns +// the offset just past the last one kept. +// +// It always keeps the first message, however large. Dropping it would leave the +// cursor unable to advance, and since an unconsumed range now schedules a +// workflow task, a stream holding one oversized message would wake the workflow +// forever without ever delivering anything. +func CapByBytes( + messages []*streamlib.StreamMessage, + from int64, + maxBytes int, +) ([]*streamlib.StreamMessage, int64) { + if len(messages) == 0 { + return messages, from + } + + total := 0 + for i, m := range messages { + total += proto.Size(m) + if total > maxBytes && i > 0 { + return messages[:i], from + int64(i) + } + } + return messages, from + int64(len(messages)) +} + +// checkBatchBytes rejects an append that is too large to store or too large to +// ever hand back. +// +// The per-message bound matters on its own: a message is never split, so one +// that exceeds a consumer's byte budget can never be delivered, and CapByBytes +// would hand it over alone forever rather than reject it. The batch bound is +// the storage side, since a batch is written as a single node. +func checkBatchBytes(messages []*streamlib.StreamMessage) error { + total := 0 + for i, m := range messages { + size := proto.Size(m) + if size > MaxMessageBytes { + return serviceerror.NewInvalidArgumentf( + "message %d is %d bytes, over the %d byte limit", i, size, MaxMessageBytes) + } + total += size + } + if total > MaxBatchBytes { + return serviceerror.NewInvalidArgumentf( + "batch is %d bytes, over the %d byte limit", total, MaxBatchBytes) + } + return nil +} diff --git a/chasm/lib/stream/messages_test.go b/chasm/lib/stream/messages_test.go new file mode 100644 index 00000000000..41a23f4620f --- /dev/null +++ b/chasm/lib/stream/messages_test.go @@ -0,0 +1,47 @@ +package stream + +import ( + "testing" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" +) + +func sized(n int, bytes int) []*streampb.StreamMessage { + out := make([]*streampb.StreamMessage, n) + for i := range out { + out[i] = &streampb.StreamMessage{ + Body: &commonpb.Payload{Data: make([]byte, bytes)}, + Kind: streampb.STREAM_MESSAGE_KIND_DATA, + } + } + return out +} + +func TestCapByBytesTrimsToAPrefix(t *testing.T) { + messages, next := CapByBytes(sized(10, 100), 4, 250) + require.Len(t, messages, 2) + require.Equal(t, int64(6), next, "the recorded range must cover exactly what was kept") +} + +func TestCapByBytesKeepsEverythingUnderBudget(t *testing.T) { + messages, next := CapByBytes(sized(3, 10), 0, 1<<20) + require.Len(t, messages, 3) + require.Equal(t, int64(3), next) +} + +// A single oversized message must still go out. Held back it would stall the +// cursor, and an unconsumed range schedules a workflow task, so the workflow +// would wake forever and never receive anything. +func TestCapByBytesAlwaysDeliversTheFirstMessage(t *testing.T) { + messages, next := CapByBytes(sized(3, 5000), 7, 10) + require.Len(t, messages, 1) + require.Equal(t, int64(8), next) +} + +func TestCapByBytesOnAnEmptyRun(t *testing.T) { + messages, next := CapByBytes(nil, 9, 100) + require.Empty(t, messages) + require.Equal(t, int64(9), next) +} diff --git a/chasm/lib/stream/proto/v1/message.proto b/chasm/lib/stream/proto/v1/message.proto new file mode 100644 index 00000000000..900ab36278a --- /dev/null +++ b/chasm/lib/stream/proto/v1/message.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.stream.proto.v1; + +import "temporal/api/common/v1/message.proto"; + +option go_package = "go.temporal.io/server/chasm/lib/stream/gen/streampb;streampb"; + +enum StreamMessageKind { + STREAM_MESSAGE_KIND_UNSPECIFIED = 0; + STREAM_MESSAGE_KIND_DATA = 1; + // Producer signalling a delivery boundary. Carries no body and consumes an + // offset like any other message. A consumer uses it to end a turn without + // waiting out an idle timeout. + STREAM_MESSAGE_KIND_FLUSH = 2; +} + +message StreamMessage { + temporal.api.common.v1.Payload body = 1; + // Producer-supplied provenance. The server does not populate this today. + map metadata = 2; + string topic = 3; + // Position within this topic. The global offset orders the whole stream; + // this lets a consumer reason about one topic without decoding the rest. + int64 topic_sequence = 4; + StreamMessageKind kind = 5; + + // Position in the whole stream, set on read and never stored. A consumer + // that resumes at message granularity needs it, and a topic-filtered read + // leaves gaps that make it underivable from the response alone. + int64 offset = 6; +} + +// One append is one batch, and one batch is one log node. The server stores +// this serialized and opaque; it decodes only to trim a partial first page or +// to apply a topic filter. +message StreamMessageBatch { + repeated StreamMessage messages = 1; +} diff --git a/chasm/lib/stream/proto/v1/request_response.proto b/chasm/lib/stream/proto/v1/request_response.proto new file mode 100644 index 00000000000..6a7cc7417f0 --- /dev/null +++ b/chasm/lib/stream/proto/v1/request_response.proto @@ -0,0 +1,364 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.stream.proto.v1; + +import "chasm/lib/stream/proto/v1/message.proto"; +import "chasm/lib/stream/proto/v1/stream_state.proto"; +import "temporal/api/common/v1/message.proto"; + +option go_package = "go.temporal.io/server/chasm/lib/stream/gen/streampb;streampb"; + +// The frontend-facing shapes are defined here rather than in the public API +// because streams have no public API yet. Keeping them in a nested +// frontend_request mirrors the other CHASM libraries, so promoting them later +// is a package move rather than a redesign. + +message CreateStreamInput { + string namespace = 1; + string stream_id = 2; + StreamLifecycle lifecycle = 3; +} + +message CreateStreamOutput { + string run_id = 1; +} + +message AddMessagesInput { + string namespace = 1; + string stream_id = 2; + // Optional. Supplying it skips resolving the stream's current run, which is + // a persistence lookup on every call. CreateStream returns it. + string run_id = 9; + repeated StreamMessage messages = 3; + + // Idempotency, all optional. Supply a producer identity and sequence, or an + // expected offset, or neither and accept at-least-once. + string producer_id = 4; + int64 sequence = 5; + // Guarded by use_expected_offset because proto3 optional is not supported + // by this repo's helper generator. + int64 expected_offset = 6; + bool use_expected_offset = 8; + + int64 owner_epoch = 7; +} + +message AddMessagesOutput { + int64 first_offset = 1; + int64 next_offset = 2; + int64 count = 3; + // True when a retry matched a recorded sequence and nothing was appended. + bool deduplicated = 4; +} + +message FinishWritingInput { + string namespace = 1; + string stream_id = 2; + string producer_id = 3; +} + +message FinishWritingOutput {} + +// Registers a Workflow as a consumer of a stream it owns. The cursor lands in +// the Workflow's own state, so from then on each of its Workflow Tasks carries +// the next range and records what it consumed. +message SubscribeWorkflowInput { + string namespace = 1; + string workflow_id = 2; + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + string owner_run_id = 6; + + // Name of the stream within the Workflow, for a stream it owns. + string stream_name = 3; + // Id of a standalone stream in another execution. Exactly one of this and + // stream_name is set. + string stream_id = 5; + // Where to start. Resolved here rather than at delivery, so the first + // recorded range starts from a fact instead of a reading. + int64 start_offset = 4; +} + +message SubscribeWorkflowOutput { + int64 start_offset = 1; +} + +message PollMessagesInput { + string namespace = 1; + string stream_id = 2; + // Optional, as on AddMessagesInput. + string run_id = 7; + int64 from_offset = 3; + int32 max_messages = 4; + // Filters by exact topic. Offsets are assigned over the unfiltered stream, so + // next_offset advances past filtered-out messages too. + repeated string topics = 5; + + // When set and the reader is caught up, block until something arrives, the + // stream closes, or the server's long-poll timeout elapses. A timeout returns + // an empty response rather than an error, so the caller simply polls again. + bool wait_new_messages = 6; +} + +message PollMessagesOutput { + repeated StreamMessage messages = 1; + int64 next_offset = 2; + int64 head_offset = 3; + bool closed = 4; + temporal.api.common.v1.Payload close_reason = 5; +} + +message DescribeStreamInput { + string namespace = 1; + string stream_id = 2; +} + +// A stream a workflow owns lives inside that workflow's execution, so it has +// no standalone id to address it by. It is named by its owner and its name +// instead, and routed on the owner. +message PollWorkflowMessagesInput { + string namespace = 1; + string workflow_id = 2; + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + string owner_run_id = 8; + + // Empty means the workflow's default output stream. + string stream_name = 3; + int64 from_offset = 4; + int32 max_messages = 5; + // Filters as on PollMessagesInput. + repeated string topics = 6; + bool wait_new_messages = 7; +} + +message DescribeWorkflowStreamInput { + string namespace = 1; + string workflow_id = 2; + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + string owner_run_id = 4; + + string stream_name = 3; +} + +// Appending to a stream a workflow owns, from outside that workflow. The +// workflow's own publishes ride its Workflow Task instead. +message AddWorkflowMessagesInput { + string namespace = 1; + string workflow_id = 2; + // Optional. Pins to one run, so a caller that has continued as new is not + // silently redirected to the successor's stream, which starts empty and at + // offset zero. Empty means whichever run is current. + string owner_run_id = 7; + + // Empty means the workflow's default output stream. + string stream_name = 3; + repeated StreamMessage messages = 4; + // Optional idempotency, as on AddMessagesInput. + string producer_id = 5; + int64 sequence = 6; +} + +message DescribeStreamOutput { + StreamState state = 1; +} + +message CloseStreamInput { + string namespace = 1; + string stream_id = 2; + temporal.api.common.v1.Payload reason = 3; +} + +message CloseStreamOutput {} + +message TruncateStreamInput { + string namespace = 1; + string stream_id = 2; + int64 new_base_offset = 3; +} + +message TruncateStreamOutput {} + +message DeleteStreamInput { + string namespace = 1; + string stream_id = 2; +} + +message DeleteStreamOutput {} + +message CreateStreamRequest { + string namespace_id = 1; + CreateStreamInput frontend_request = 2; +} +message CreateStreamResponse { + CreateStreamOutput frontend_response = 1; +} + +message AddMessagesRequest { + string namespace_id = 1; + AddMessagesInput frontend_request = 2; +} +message AddMessagesResponse { + AddMessagesOutput frontend_response = 1; +} + +message FinishWritingRequest { + string namespace_id = 1; + FinishWritingInput frontend_request = 2; +} +message FinishWritingResponse { + FinishWritingOutput frontend_response = 1; +} + +message SubscribeWorkflowRequest { + string namespace_id = 1; + SubscribeWorkflowInput frontend_request = 2; +} +message SubscribeWorkflowResponse { + SubscribeWorkflowOutput frontend_response = 1; +} + +message PollMessagesRequest { + string namespace_id = 1; + PollMessagesInput frontend_request = 2; +} +message PollMessagesResponse { + PollMessagesOutput frontend_response = 1; +} + +message DescribeStreamRequest { + string namespace_id = 1; + DescribeStreamInput frontend_request = 2; +} +message DescribeStreamResponse { + DescribeStreamOutput frontend_response = 1; +} + +message PollWorkflowMessagesRequest { + string namespace_id = 1; + PollWorkflowMessagesInput frontend_request = 2; +} +message PollWorkflowMessagesResponse { + PollMessagesOutput frontend_response = 1; +} + +message DescribeWorkflowStreamRequest { + string namespace_id = 1; + DescribeWorkflowStreamInput frontend_request = 2; +} +message DescribeWorkflowStreamResponse { + DescribeStreamOutput frontend_response = 1; +} + +// Registering a consumer on a stream in another execution. Split out from +// SubscribeWorkflow because the two halves live on different shards: the pin +// goes on the stream, the cursor goes on the consuming workflow, and a handler +// can only reach the shard it was routed to. +message RegisterStreamConsumerInput { + string namespace = 1; + string stream_id = 2; + // The workflow that will consume, which names the pin. + string consumer_workflow_id = 3; + // Negative means from wherever the stream is when the pin is taken. Resolved + // here, where the frontier is, and returned so the cursor records a fact. + int64 start_offset = 4; +} + +message RegisterStreamConsumerOutput { + int64 start_offset = 1; + // What the consumer needs to address the log, all decided by the stream. + string collection_id = 2; + int64 bucket_size = 3; + int64 known_head = 4; +} + +// Telling one consumer that the frontier moved. Routed to the consumer, which +// is not where the stream lives. +message AdvanceConsumerHeadInput { + string namespace = 1; + string workflow_id = 2; + // The run that subscribed. A successor from continue-as-new holds no cursor + // for this stream, so pushing the frontier at it would land nowhere. + string owner_run_id = 5; + string stream_id = 3; + int64 head_offset = 4; +} + +message AdvanceConsumerHeadOutput {} + +message AddWorkflowMessagesRequest { + string namespace_id = 1; + AddWorkflowMessagesInput frontend_request = 2; +} +message AddWorkflowMessagesResponse { + AddMessagesOutput frontend_response = 1; +} + +message RegisterStreamConsumerRequest { + string namespace_id = 1; + RegisterStreamConsumerInput frontend_request = 2; +} +message RegisterStreamConsumerResponse { + RegisterStreamConsumerOutput frontend_response = 1; +} + +message AdvanceConsumerHeadRequest { + string namespace_id = 1; + AdvanceConsumerHeadInput frontend_request = 2; +} +message AdvanceConsumerHeadResponse { + AdvanceConsumerHeadOutput frontend_response = 1; +} + +message CloseStreamRequest { + string namespace_id = 1; + CloseStreamInput frontend_request = 2; +} +message CloseStreamResponse { + CloseStreamOutput frontend_response = 1; +} + +message TruncateStreamRequest { + string namespace_id = 1; + TruncateStreamInput frontend_request = 2; +} +message TruncateStreamResponse { + TruncateStreamOutput frontend_response = 1; +} + +message ListStreamsInput { + string namespace = 1; + int32 page_size = 2; + bytes next_page_token = 3; + string query = 4; +} + +message StreamListEntry { + string stream_id = 1; + string run_id = 2; +} + +message ListStreamsOutput { + repeated StreamListEntry streams = 1; + bytes next_page_token = 2; +} + +message ListStreamsRequest { + string namespace_id = 1; + ListStreamsInput frontend_request = 2; +} +message ListStreamsResponse { + ListStreamsOutput frontend_response = 1; +} + +message DeleteStreamRequest { + string namespace_id = 1; + DeleteStreamInput frontend_request = 2; +} +message DeleteStreamResponse { + DeleteStreamOutput frontend_response = 1; +} diff --git a/chasm/lib/stream/proto/v1/service.proto b/chasm/lib/stream/proto/v1/service.proto new file mode 100644 index 00000000000..bdd0a918551 --- /dev/null +++ b/chasm/lib/stream/proto/v1/service.proto @@ -0,0 +1,92 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.stream.proto.v1; + +import "chasm/lib/stream/proto/v1/request_response.proto"; +import "temporal/server/api/common/v1/api_category.proto"; +import "temporal/server/api/routing/v1/extension.proto"; + +option go_package = "go.temporal.io/server/chasm/lib/stream/gen/streampb;streampb"; + +service StreamService { + rpc CreateStream(CreateStreamRequest) returns (CreateStreamResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc AddMessages(AddMessagesRequest) returns (AddMessagesResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc FinishWriting(FinishWritingRequest) returns (FinishWritingResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc SubscribeWorkflow(SubscribeWorkflowRequest) returns (SubscribeWorkflowResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.workflow_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc PollMessages(PollMessagesRequest) returns (PollMessagesResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_LONG_POLL; + } + + rpc DescribeStream(DescribeStreamRequest) returns (DescribeStreamResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + // Routed on the owner, because the stream it reads has no id of its own. + rpc PollWorkflowMessages(PollWorkflowMessagesRequest) returns (PollWorkflowMessagesResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.workflow_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_LONG_POLL; + } + + rpc DescribeWorkflowStream(DescribeWorkflowStreamRequest) returns (DescribeWorkflowStreamResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.workflow_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc AddWorkflowMessages(AddWorkflowMessagesRequest) returns (AddWorkflowMessagesResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.workflow_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + // Internal. History calls these on itself to reach a shard it does not own, + // which is the only way a step that spans two executions can work on a + // cluster with more than one history host. + rpc RegisterStreamConsumer(RegisterStreamConsumerRequest) returns (RegisterStreamConsumerResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc AdvanceConsumerHead(AdvanceConsumerHeadRequest) returns (AdvanceConsumerHeadResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.workflow_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc CloseStream(CloseStreamRequest) returns (CloseStreamResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc TruncateStream(TruncateStreamRequest) returns (TruncateStreamResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + // Served on the frontend only: it queries visibility rather than a stream, + // so there is no business ID to route on and nothing for a shard to answer. + rpc ListStreams(ListStreamsRequest) returns (ListStreamsResponse) { + option (temporal.server.api.routing.v1.routing).random = true; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc DeleteStream(DeleteStreamRequest) returns (DeleteStreamResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.stream_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } +} diff --git a/chasm/lib/stream/proto/v1/stream_state.proto b/chasm/lib/stream/proto/v1/stream_state.proto new file mode 100644 index 00000000000..81c878947a6 --- /dev/null +++ b/chasm/lib/stream/proto/v1/stream_state.proto @@ -0,0 +1,123 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.stream.proto.v1; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "temporal/api/common/v1/message.proto"; + +option go_package = "go.temporal.io/server/chasm/lib/stream/gen/streampb;streampb"; + +// Size is O(producers + consumers), never O(messages). Payload bytes live in +// the log, not here, which is what keeps this off the CHASM partial-read path. +message StreamState { + // Visibility frontier. Readers never observe an offset at or past this. + int64 head_offset = 1; + // Truncation floor. Offsets below this are gone. + int64 base_offset = 2; + // Chains log nodes so a stale node from an abandoned append is rejected on + + bool closed = 4; + temporal.api.common.v1.Payload close_reason = 5; + + // Bumped on ownership change so a stale producer's write fails. + int64 owner_epoch = 6; + + // Immutable once set. Offsets roll to a new log tree every bucket_size so no + // single storage partition grows with the stream. + int64 bucket_size = 7; + + // Identity of the log this stream writes to. Bucket trees are derived from + // it, so there is no per-bucket index to store. + string collection_id = 8; + + map producers = 9; + map consumers = 10; + + StreamLifecycle lifecycle = 11; + + // Set when a successor run takes ownership, so an in-flight poll can follow + // the chain instead of stalling on a superseded run. + string redirect_run_id = 12; + + // Wall-clock close time, used to schedule retention deletion. + google.protobuf.Timestamp close_time = 13; +} + +message ProducerCursor { + int64 seq = 1; + int64 first_offset = 2; + int64 count = 3; + // Distinguishes a genuine retry from a client reusing a sequence with + // different content, which must be rejected rather than deduplicated. + bytes content_hash = 4; + // Set by FinishWriting. Ends this producer's writes without closing the + // stream for anyone else. + bool fenced = 5; +} + +message ConsumerCursor { + string workflow_id = 1; + string run_id = 2; + // How far this consumer has read. Used to decide whether it needs waking, + // not to decide what the stream may drop. + int64 offset = 3; + bool active = 4; + // Set when the consumer is a workflow in another execution, which is the + // only case that has to be told the frontier moved. A workflow consuming a + // stream it owns sees that while closing its own transaction. + bool external = 5; + // The oldest offset this consumer's History still depends on. + // + // It is where the subscription started, not where it has read to. A range + // this consumer already consumed is recorded in its History and has to be + // re-readable for the workflow to replay, so bytes below the read position + // are exactly the ones a replay needs most. It stays put while the consumer + // is active and is released when it deregisters. + int64 replay_floor = 6; +} + +// A consuming Workflow's position in a stream. This lives in the consuming +// Workflow's own state rather than on the stream, so advancing it commits in +// the same transaction as the WorkflowTaskCompleted event that records the +// range. Keeping it on the stream would make the advance a cross-execution +// write, and a crash between the two would either redeliver or skip. +message WorkflowStreamCursor { + string stream_id = 1; + // Enough to address the log without reading the stream component first. + string collection_id = 2; + int64 bucket_size = 3; + + // Next offset to deliver. + int64 offset = 4; + + // The stream's frontier as of the last delivery. A workflow consuming a + // stream in another execution cannot read the real frontier while closing its + // own transaction, so this is what tells it a capped slice left more behind. + int64 known_head = 8; + + // Set when the stream lives in another execution, which decides whether the + // frontier is read locally or taken from known_head. + bool external = 9; + + // The range attached to the Workflow Task currently in flight. Recorded on + // the event that closes that task, then folded into offset. An empty range + // is still recorded: a task where the subscription saw nothing is a fact + // replay has to reproduce. + int64 pending_from = 5; + int64 pending_to = 6; + bool has_pending = 7; + + // Where this subscription began reading. Retained separately from offset, + // which advances, because replay has to tell "committed nothing yet" apart + // from "the events recording what was committed are not in this page". + int64 start_offset = 10; +} + +message StreamLifecycle { + // How long a closed stream stays readable before it is deleted. + google.protobuf.Duration retention = 1; + // Cap on readable messages. Older whole buckets are reclaimed once the floor + // passes them, so a capped stream has bounded storage. + int64 max_items = 2; +} diff --git a/chasm/lib/stream/proto/v1/tasks.proto b/chasm/lib/stream/proto/v1/tasks.proto new file mode 100644 index 00000000000..c4b8e5e6c6a --- /dev/null +++ b/chasm/lib/stream/proto/v1/tasks.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.stream.proto.v1; + +option go_package = "go.temporal.io/server/chasm/lib/stream/gen/streampb;streampb"; + +// Fires at close_time plus retention. A closed stream stays readable until +// then, which is what removes the shutdown handshake the signal-based +// implementation forces on producers and consumers. +message StreamRetentionTask {} + +// Fires after an append that leaves a registered consumer behind. Appending +// never schedules a workflow task on its own, because a stream item is data an +// execution produced rather than a decision input to it. A workflow that +// subscribed is the exception: it asked to be told, and it cannot find out any +// other way, since nothing else it does would notice the stream moved. +message StreamNotifyConsumersTask {} diff --git a/chasm/lib/stream/service/events.go b/chasm/lib/stream/service/events.go new file mode 100644 index 00000000000..0088b1eec5e --- /dev/null +++ b/chasm/lib/stream/service/events.go @@ -0,0 +1,78 @@ +package service + +import ( + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + "go.temporal.io/server/service/history/hsm" +) + +// streamSubscribedEventDefinition tells the history service how to treat the +// event a subscribe command writes. +// +// It applies nothing. The cursor lives in CHASM state, which is persisted and +// rebuilt with the execution, so replication and reset have nothing to +// reconstruct from the event. It exists so the command has an event at all: +// every SDK matches issued commands against command-generated events in order, +// and a command producing none puts that matching out of step. +type streamSubscribedEventDefinition struct{} + +func (streamSubscribedEventDefinition) Type() enumspb.EventType { + return enumspb.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED +} + +// Subscribing does not itself give the workflow anything to decide on. The +// range that follows does, and that wakes the workflow through its cursor. +func (streamSubscribedEventDefinition) IsWorkflowTaskTrigger() bool { return false } + +func (streamSubscribedEventDefinition) Apply(*hsm.Node, *historypb.HistoryEvent) error { + return nil +} + +// A command event, so never reapplied onto another branch: a workflow that +// still wants the subscription issues the command again on the new one. +func (streamSubscribedEventDefinition) CherryPick( + *hsm.Node, + *historypb.HistoryEvent, + map[enumspb.ResetReapplyExcludeType]struct{}, +) error { + return hsm.ErrNotCherryPickable +} + +// streamMessagesAddedEventDefinition tells the history service how to treat +// the event a publish command writes. +// +// Like the subscription event it applies nothing: the stream's frontier is +// CHASM state committed with the workflow task, and the bodies are in the +// stream's own log. What the event carries is the offset range, which is what +// lets anyone reading History find the batch without History having held it. +type streamMessagesAddedEventDefinition struct{} + +func (streamMessagesAddedEventDefinition) Type() enumspb.EventType { + return enumspb.EVENT_TYPE_WORKFLOW_STREAM_MESSAGES_ADDED +} + +// A workflow publishing to its own stream has nothing to be woken about. +func (streamMessagesAddedEventDefinition) IsWorkflowTaskTrigger() bool { return false } + +func (streamMessagesAddedEventDefinition) Apply(*hsm.Node, *historypb.HistoryEvent) error { + return nil +} + +// A command event, so never reapplied onto another branch. Reapplying would +// claim offsets in a log the new branch never wrote to. +func (streamMessagesAddedEventDefinition) CherryPick( + *hsm.Node, + *historypb.HistoryEvent, + map[enumspb.ResetReapplyExcludeType]struct{}, +) error { + return hsm.ErrNotCherryPickable +} + +// RegisterEventDefinitions makes the stream events known to the history +// service. +func RegisterEventDefinitions(reg *hsm.Registry) error { + if err := reg.RegisterEventDefinition(streamSubscribedEventDefinition{}); err != nil { + return err + } + return reg.RegisterEventDefinition(streamMessagesAddedEventDefinition{}) +} diff --git a/chasm/lib/stream/service/frontend.go b/chasm/lib/stream/service/frontend.go new file mode 100644 index 00000000000..34d6e18e644 --- /dev/null +++ b/chasm/lib/stream/service/frontend.go @@ -0,0 +1,231 @@ +package service + +import ( + "context" + + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "google.golang.org/protobuf/types/known/emptypb" +) + +// FrontendHandler serves StreamService on the frontend. It resolves the +// namespace name to an ID and forwards to the history shard that owns the +// stream; the layered client does the routing from the business ID. +type FrontendHandler struct { + streampb.UnimplementedStreamServiceServer + + client streampb.StreamServiceClient + namespaceRegistry namespace.Registry + logger log.Logger +} + +func NewFrontendHandler( + client streampb.StreamServiceClient, + namespaceRegistry namespace.Registry, + logger log.Logger, +) *FrontendHandler { + return &FrontendHandler{ + client: client, + namespaceRegistry: namespaceRegistry, + logger: logger, + } +} + +func (h *FrontendHandler) namespaceID(name string) (string, error) { + if name == "" { + return "", serviceerror.NewInvalidArgument("namespace is required") + } + id, err := h.namespaceRegistry.GetNamespaceID(namespace.Name(name)) + if err != nil { + return "", err + } + return id.String(), nil +} + +func (h *FrontendHandler) CreateStream( + ctx context.Context, req *streampb.CreateStreamRequest, +) (*streampb.CreateStreamResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.CreateStream(ctx, &streampb.CreateStreamRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) AddMessages( + ctx context.Context, req *streampb.AddMessagesRequest, +) (*streampb.AddMessagesResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.AddMessages(ctx, &streampb.AddMessagesRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) FinishWriting( + ctx context.Context, req *streampb.FinishWritingRequest, +) (*streampb.FinishWritingResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.FinishWriting(ctx, &streampb.FinishWritingRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) SubscribeWorkflow( + ctx context.Context, req *streampb.SubscribeWorkflowRequest, +) (*streampb.SubscribeWorkflowResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.SubscribeWorkflow(ctx, &streampb.SubscribeWorkflowRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) PollMessages( + ctx context.Context, req *streampb.PollMessagesRequest, +) (*streampb.PollMessagesResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.PollMessages(ctx, &streampb.PollMessagesRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) PollWorkflowMessages( + ctx context.Context, req *streampb.PollWorkflowMessagesRequest, +) (*streampb.PollWorkflowMessagesResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.PollWorkflowMessages(ctx, &streampb.PollWorkflowMessagesRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) DescribeWorkflowStream( + ctx context.Context, req *streampb.DescribeWorkflowStreamRequest, +) (*streampb.DescribeWorkflowStreamResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.DescribeWorkflowStream(ctx, &streampb.DescribeWorkflowStreamRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) AddWorkflowMessages( + ctx context.Context, req *streampb.AddWorkflowMessagesRequest, +) (*streampb.AddWorkflowMessagesResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.AddWorkflowMessages(ctx, &streampb.AddWorkflowMessagesRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) DescribeStream( + ctx context.Context, req *streampb.DescribeStreamRequest, +) (*streampb.DescribeStreamResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.DescribeStream(ctx, &streampb.DescribeStreamRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) CloseStream( + ctx context.Context, req *streampb.CloseStreamRequest, +) (*streampb.CloseStreamResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.CloseStream(ctx, &streampb.CloseStreamRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) TruncateStream( + ctx context.Context, req *streampb.TruncateStreamRequest, +) (*streampb.TruncateStreamResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.TruncateStream(ctx, &streampb.TruncateStreamRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +func (h *FrontendHandler) DeleteStream( + ctx context.Context, req *streampb.DeleteStreamRequest, +) (*streampb.DeleteStreamResponse, error) { + id, err := h.namespaceID(req.GetFrontendRequest().GetNamespace()) + if err != nil { + return nil, err + } + return h.client.DeleteStream(ctx, &streampb.DeleteStreamRequest{ + NamespaceId: id, FrontendRequest: req.GetFrontendRequest(), + }) +} + +// ListStreams answers from visibility rather than from any one stream, so it +// does not route to a shard and is served here rather than on the history side. +func (h *FrontendHandler) ListStreams( + ctx context.Context, req *streampb.ListStreamsRequest, +) (*streampb.ListStreamsResponse, error) { + in := req.GetFrontendRequest() + if in.GetNamespace() == "" { + return nil, serviceerror.NewInvalidArgument("namespace is required") + } + + pageSize := int(in.GetPageSize()) + if pageSize <= 0 || pageSize > stream.MaxListPageSize { + pageSize = stream.MaxListPageSize + } + + resp, err := chasm.ListExecutions[*stream.Stream, *emptypb.Empty](ctx, &chasm.ListExecutionsRequest{ + NamespaceName: in.GetNamespace(), + PageSize: pageSize, + NextPageToken: in.GetNextPageToken(), + Query: in.GetQuery(), + }) + if err != nil { + return nil, err + } + + entries := make([]*streampb.StreamListEntry, 0, len(resp.Executions)) + for _, e := range resp.Executions { + entries = append(entries, &streampb.StreamListEntry{ + StreamId: e.BusinessID, + RunId: e.RunID, + }) + } + return &streampb.ListStreamsResponse{ + FrontendResponse: &streampb.ListStreamsOutput{ + Streams: entries, + NextPageToken: resp.NextPageToken, + }, + }, nil +} diff --git a/chasm/lib/stream/service/fx.go b/chasm/lib/stream/service/fx.go new file mode 100644 index 00000000000..06ad4ffa779 --- /dev/null +++ b/chasm/lib/stream/service/fx.go @@ -0,0 +1,35 @@ +package service + +import ( + "go.temporal.io/server/chasm" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "go.uber.org/fx" +) + +var HistoryModule = fx.Module( + "stream-history", + fx.Provide( + // Routes a call to the host owning a shard. History needs it too, not + // just the frontend: a step that spans two executions has to reach a + // shard this host may not own. + streampb.NewStreamServiceLayeredClient, + newHandler, + newRetentionTaskHandler, + newNotifyConsumersTaskHandler, + newLibrary, + ), + fx.Invoke(func(l *library, registry *chasm.Registry) error { + return registry.Register(l) + }), + fx.Invoke(RegisterEventDefinitions), +) + +var FrontendModule = fx.Module( + "stream-frontend", + fx.Provide(streampb.NewStreamServiceLayeredClient), + fx.Provide(NewFrontendHandler), + fx.Provide(newComponentOnlyLibrary), + fx.Invoke(func(l *componentOnlyLibrary, registry *chasm.Registry) error { + return registry.Register(l) + }), +) diff --git a/chasm/lib/stream/service/handler.go b/chasm/lib/stream/service/handler.go new file mode 100644 index 00000000000..41da753d9be --- /dev/null +++ b/chasm/lib/stream/service/handler.go @@ -0,0 +1,839 @@ +package service + +import ( + "context" + "sync" + + "github.com/dgryski/go-farm" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/common" + "go.temporal.io/server/common/contextutil" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/service/history/shard" +) + +type handler struct { + streampb.UnimplementedStreamServiceServer + + shardController shard.Controller + namespaceRegistry namespace.Registry + logger log.Logger + + // Routes a call to the host that owns a shard. A step spanning two + // executions cannot resolve both through the local controller, which + // refuses a shard this host does not own, so the far half goes back out + // through the service and lands wherever it belongs. + routed streampb.StreamServiceClient + + // Appends to one stream are serialized here. The node has to be durable + // before the frontier advances, which means writing it outside the + // transition that advances the frontier, and two concurrent writers could + // then commit in a different order than they wrote. Whichever node carried + // the higher transaction ID would win the read regardless of which writer + // actually committed. + // + // Serializing removes the interleaving. It is a stopgap: the real fix is to + // stage the node inside the CHASM transaction so write and commit order + // cannot diverge. Until then this also means appends are only safe within + // one process, which holds because these RPCs route to the shard owner. + // + // Striped rather than one lock per stream. The key comes from request input + // before the stream is known to exist, so a per-stream map would grow once + // per distinct id a caller names, including ids that resolve to nothing. + // Unrelated streams sharing a stripe only serialize with each other. + appendLk [appendStripes]sync.Mutex +} + +func newHandler( + shardController shard.Controller, + namespaceRegistry namespace.Registry, + logger log.Logger, + routed streampb.StreamServiceClient, +) *handler { + return &handler{ + shardController: shardController, + namespaceRegistry: namespaceRegistry, + logger: logger, + routed: routed, + } +} + +func streamKey(namespaceID, streamID string) string { + return namespaceID + "/" + streamID +} + +// withCallerInfo tags the context so the stream's direct persistence calls are +// attributed to the namespace that caused them. Without it they carry no caller +// name, which means they escape namespace rate limiting and priority as well as +// going uncounted in per-namespace metrics. The RPC path sets this via +// interceptors; calls made outside a request handler have to set it themselves. +func (h *handler) withCallerInfo(ctx context.Context, namespaceID string) context.Context { + name, err := h.namespaceRegistry.GetNamespaceName(namespace.ID(namespaceID)) + if err != nil { + return ctx + } + return headers.SetCallerInfo(ctx, headers.NewCallerInfo( + name.String(), headers.CallerTypeAPI, "")) +} + +func (h *handler) lockStream(namespaceID, streamID string) func() { + mu := &h.appendLk[appendStripe(streamKey(namespaceID, streamID))] + mu.Lock() + return mu.Unlock +} + +// Sized well above the per-host stream count that would make collisions matter. +// Appends to one stream are serialized anyway, so a collision costs only the +// unrelated stream's concurrency, never correctness. +const appendStripes = 2048 + +func appendStripe(key string) uint32 { + return farm.Fingerprint32([]byte(key)) % appendStripes +} + +// refFor builds a reference to a stream. A supplied run ID lets the engine skip +// resolving the current run, which is otherwise a persistence lookup on every +// call and dominates the cost of an otherwise cheap read. +func refFor(namespaceID, streamID string) chasm.ComponentRef { + return refForRun(namespaceID, streamID, "") +} + +func refForRun(namespaceID, streamID, runID string) chasm.ComponentRef { + return chasm.NewComponentRef[*stream.Stream](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: streamID, + RunID: runID, + }) +} + +// workflowRef builds a reference to the execution that owns an attached +// stream. An attached stream is a subcomponent, so it has no id of its own and +// everything about it is reached through its owner. +// +// An empty runID means the current run. A caller that supplies one is pinning: +// an owned stream does not carry across continue-as-new, so the successor's +// stream of the same name is a different, empty one, and a caller that meant +// the predecessor would otherwise be redirected to it without being told. +func workflowRef(namespaceID, workflowID, runID string) chasm.ComponentRef { + return chasm.NewComponentRef[*chasmworkflow.Workflow](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: workflowID, + RunID: runID, + }) +} + +// ownedStreamName resolves a name the caller left empty the same way a publish +// command does, so a reader addresses the default stream by omission just as a +// writer creates it by omission. +func ownedStreamName(name string) string { + if name == "" { + return chasmworkflow.DefaultStreamName + } + return name +} + +func (h *handler) CreateStream( + ctx context.Context, + req *streampb.CreateStreamRequest, +) (*streampb.CreateStreamResponse, error) { + in := req.GetFrontendRequest() + if in.GetStreamId() == "" { + return nil, serviceerror.NewInvalidArgument("stream id is required") + } + + result, err := chasm.StartExecution( + ctx, + chasm.ExecutionKey{NamespaceID: req.GetNamespaceId(), BusinessID: in.GetStreamId()}, + func(mctx chasm.MutableContext, input *streampb.CreateStreamInput) (*stream.Stream, error) { + return stream.NewStream(mctx, stream.NewStreamRequest{ + CollectionID: mctx.ExecutionKey().RunID, + Lifecycle: input.GetLifecycle(), + }) + }, + in, + ) + if err != nil { + return nil, err + } + return &streampb.CreateStreamResponse{ + FrontendResponse: &streampb.CreateStreamOutput{RunId: result.ExecutionKey.RunID}, + }, nil +} + +func (h *handler) AddMessages( + ctx context.Context, + req *streampb.AddMessagesRequest, +) (*streampb.AddMessagesResponse, error) { + in := req.GetFrontendRequest() + if len(in.GetMessages()) == 0 { + return nil, serviceerror.NewInvalidArgument("no messages to append") + } + + unlock := h.lockStream(req.GetNamespaceId(), in.GetStreamId()) + defer unlock() + + ctx = h.withCallerInfo(ctx, req.GetNamespaceId()) + + ref := refForRun(req.GetNamespaceId(), in.GetStreamId(), in.GetRunId()) + state, err := chasm.ReadComponent(ctx, ref, (*stream.Stream).Snapshot, struct{}{}) + if err != nil { + return nil, err + } + + addReq := stream.AddMessagesRequest{ + Messages: in.GetMessages(), + ProducerID: in.GetProducerId(), + Sequence: in.GetSequence(), + OwnerEpoch: in.GetOwnerEpoch(), + } + if in.GetUseExpectedOffset() { + expected := in.GetExpectedOffset() + addReq.ExpectedOffset = &expected + } else { + // Without a caller-supplied expectation, pin to the head we just read. + // The transition then fails rather than appending at an offset whose + // node we did not write. + head := state.GetHeadOffset() + addReq.ExpectedOffset = &head + } + + result, _, err := chasm.UpdateComponent(ctx, ref, (*stream.Stream).AddMessages, addReq) + if err != nil { + return nil, err + } + + return &streampb.AddMessagesResponse{ + FrontendResponse: &streampb.AddMessagesOutput{ + FirstOffset: result.FirstOffset, + NextOffset: result.NextOffset, + Count: result.Count, + Deduplicated: result.Deduplicated, + }, + }, nil +} + +// AddWorkflowMessages appends to a stream a workflow owns, from outside that +// workflow. +// +// The workflow's own publishes ride its Workflow Task and cost no transition of +// their own. This producer is off-shard, so it pays one transition on the +// owning execution per batch, and batching is what keeps that cheap. It is the +// path a model activity streaming tokens takes, where the workflow is only +// bracketing what the activity produces. +func (h *handler) AddWorkflowMessages( + ctx context.Context, + req *streampb.AddWorkflowMessagesRequest, +) (*streampb.AddWorkflowMessagesResponse, error) { + in := req.GetFrontendRequest() + if len(in.GetMessages()) == 0 { + return nil, serviceerror.NewInvalidArgument("no messages to append") + } + + name := ownedStreamName(in.GetStreamName()) + + // Keyed on the owner and the name, which is what identifies the log here. + unlock := h.lockStream(req.GetNamespaceId(), in.GetWorkflowId()+"/"+name) + defer unlock() + + ctx = h.withCallerInfo(ctx, req.GetNamespaceId()) + + ref := workflowRef(req.GetNamespaceId(), in.GetWorkflowId(), in.GetOwnerRunId()) + + state, err := chasm.ReadComponent(ctx, ref, + func(wf *chasmworkflow.Workflow, cctx chasm.Context, streamName string) (*streampb.StreamState, error) { + return wf.OwnedStreamState(cctx, streamName) + }, name) + if err != nil { + return nil, err + } + if state == nil { + // Nothing has published yet, so the collection id and bucket size this + // write needs do not exist. Creating the stream is a transition, and + // only the first writer ever pays it. + state, _, err = chasm.UpdateComponent(ctx, ref, + (*chasmworkflow.Workflow).EnsureOwnedStream, name) + if err != nil { + return nil, err + } + } + + head := state.GetHeadOffset() + addReq := stream.AddMessagesRequest{ + Messages: in.GetMessages(), + ProducerID: in.GetProducerId(), + Sequence: in.GetSequence(), + // Pinned to the head just read, so a workflow task that published + // between the read and the commit fails this append rather than + // letting it claim offsets whose node it did not write. + ExpectedOffset: &head, + } + + result, _, err := chasm.UpdateComponent(ctx, ref, + func(wf *chasmworkflow.Workflow, mctx chasm.MutableContext, r stream.AddMessagesRequest) (stream.AddMessagesResult, error) { + return wf.AppendToOwnedStream(mctx, name, r) + }, addReq) + if err != nil { + return nil, err + } + + return &streampb.AddWorkflowMessagesResponse{ + FrontendResponse: &streampb.AddMessagesOutput{ + FirstOffset: result.FirstOffset, + NextOffset: result.NextOffset, + Count: result.Count, + Deduplicated: result.Deduplicated, + }, + }, nil +} + +func (h *handler) FinishWriting( + ctx context.Context, + req *streampb.FinishWritingRequest, +) (*streampb.FinishWritingResponse, error) { + in := req.GetFrontendRequest() + _, _, err := chasm.UpdateComponent( + ctx, + refFor(req.GetNamespaceId(), in.GetStreamId()), + func(s *stream.Stream, mctx chasm.MutableContext, producerID string) (struct{}, error) { + return struct{}{}, s.FinishWriting(mctx, producerID) + }, + in.GetProducerId(), + ) + if err != nil { + return nil, err + } + return &streampb.FinishWritingResponse{FrontendResponse: &streampb.FinishWritingOutput{}}, nil +} + +// SubscribeWorkflow registers a workflow as a consumer of a stream it owns. +// +// The cursor is written into the workflow's own state, not the stream's, which +// is what lets every later advance commit with the event that records it. Only +// a stream the workflow owns can be subscribed here: reaching one in another +// execution needs that stream's frontier, and reading it from inside the +// consuming workflow's transaction is a separate problem. +func (h *handler) SubscribeWorkflow( + ctx context.Context, + req *streampb.SubscribeWorkflowRequest, +) (*streampb.SubscribeWorkflowResponse, error) { + in := req.GetFrontendRequest() + + if in.GetStreamId() != "" { + return h.subscribeToExternalStream(ctx, req.GetNamespaceId(), in) + } + + startOffset, _, err := chasm.UpdateComponent( + ctx, + workflowRef(req.GetNamespaceId(), in.GetWorkflowId(), in.GetOwnerRunId()), + func(wf *chasmworkflow.Workflow, mctx chasm.MutableContext, input *streampb.SubscribeWorkflowInput) (int64, error) { + return wf.SubscribeToOwnedStream( + mctx, ownedStreamName(input.GetStreamName()), input.GetStartOffset()) + }, + in, + ) + if err != nil { + return nil, err + } + + return &streampb.SubscribeWorkflowResponse{ + FrontendResponse: &streampb.SubscribeWorkflowOutput{StartOffset: startOffset}, + }, nil +} + +// subscribeToExternalStream registers a cursor against a stream in another +// execution. +// +// The pin goes on the stream before the cursor goes on the workflow, and the +// order is the guarantee: interrupted after the first write there is a pin +// holding storage nothing reads, which costs space. Interrupted after the +// other order there would be a cursor with no pin, and truncation would be free +// to take a range that cursor still points at. +func (h *handler) subscribeToExternalStream( + ctx context.Context, + namespaceID string, + in *streampb.SubscribeWorkflowInput, +) (*streampb.SubscribeWorkflowResponse, error) { + // The stream half goes out and comes back on the shard that owns it. This + // handler was routed to the consuming workflow, so the stream may well be + // somewhere else, and resolving it here would fail on any cluster with more + // than one history host. + // + // The pin still lands before the cursor, which is the guarantee: interrupted + // between them there is a pin holding storage nothing reads, which costs + // space, where the other order would leave a cursor with no pin and let + // truncation take a range it still points at. + registered, err := h.routed.RegisterStreamConsumer(ctx, &streampb.RegisterStreamConsumerRequest{ + NamespaceId: namespaceID, + FrontendRequest: &streampb.RegisterStreamConsumerInput{ + Namespace: in.GetNamespace(), + StreamId: in.GetStreamId(), + ConsumerWorkflowId: in.GetWorkflowId(), + StartOffset: in.GetStartOffset(), + }, + }) + if err != nil { + return nil, err + } + pin := registered.GetFrontendResponse() + + startOffset, _, err := chasm.UpdateComponent( + ctx, + workflowRef(namespaceID, in.GetWorkflowId(), in.GetOwnerRunId()), + func(wf *chasmworkflow.Workflow, mctx chasm.MutableContext, offset int64) (int64, error) { + return wf.SubscribeToExternalStream(mctx, chasmworkflow.ExternalStreamSubscription{ + StreamID: in.GetStreamId(), + CollectionID: pin.GetCollectionId(), + BucketSize: pin.GetBucketSize(), + StartOffset: offset, + KnownHead: pin.GetKnownHead(), + }) + }, + pin.GetStartOffset(), + ) + if err != nil { + return nil, err + } + + return &streampb.SubscribeWorkflowResponse{ + FrontendResponse: &streampb.SubscribeWorkflowOutput{StartOffset: startOffset}, + }, nil +} + +// RegisterStreamConsumer takes the pin, on the shard that owns the stream. +// +// Internal. Called by SubscribeWorkflow, which is routed to the consumer and so +// cannot reach the stream itself. It resolves a negative start offset here, +// where the frontier is, and hands back everything the cursor needs to address +// the log, so the consumer records facts rather than readings. +func (h *handler) RegisterStreamConsumer( + ctx context.Context, + req *streampb.RegisterStreamConsumerRequest, +) (*streampb.RegisterStreamConsumerResponse, error) { + in := req.GetFrontendRequest() + ctx = h.withCallerInfo(ctx, req.GetNamespaceId()) + + streamID := in.GetStreamId() + ref := refFor(req.GetNamespaceId(), streamID) + + state, err := chasm.ReadComponent(ctx, ref, (*stream.Stream).Snapshot, struct{}{}) + if err != nil { + return nil, err + } + + startOffset := in.GetStartOffset() + if startOffset < 0 { + startOffset = state.GetHeadOffset() + } + if startOffset < state.GetBaseOffset() { + return nil, serviceerror.NewFailedPreconditionf( + "offset %d is below the stream's floor of %d", startOffset, state.GetBaseOffset()) + } + + consumerID := "workflow:" + in.GetConsumerWorkflowId() + if _, _, err := chasm.UpdateComponent( + ctx, + ref, + func(s *stream.Stream, mctx chasm.MutableContext, offset int64) (struct{}, error) { + return struct{}{}, s.RegisterConsumer( + mctx, consumerID, in.GetConsumerWorkflowId(), "", offset, true) + }, + startOffset, + ); err != nil { + return nil, err + } + + return &streampb.RegisterStreamConsumerResponse{ + FrontendResponse: &streampb.RegisterStreamConsumerOutput{ + StartOffset: startOffset, + CollectionId: state.GetCollectionId(), + BucketSize: state.GetBucketSize(), + KnownHead: state.GetHeadOffset(), + }, + }, nil +} + +// AdvanceConsumerHead tells one consumer that the frontier moved, on the shard +// that owns that consumer. +// +// Internal. Called by the notify task, which runs on the stream's shard and so +// cannot reach a consumer living anywhere else. +func (h *handler) AdvanceConsumerHead( + ctx context.Context, + req *streampb.AdvanceConsumerHeadRequest, +) (*streampb.AdvanceConsumerHeadResponse, error) { + in := req.GetFrontendRequest() + ctx = h.withCallerInfo(ctx, req.GetNamespaceId()) + + if _, _, err := chasm.UpdateComponent( + ctx, + workflowRef(req.GetNamespaceId(), in.GetWorkflowId(), in.GetOwnerRunId()), + func(wf *chasmworkflow.Workflow, mctx chasm.MutableContext, at int64) (struct{}, error) { + return struct{}{}, wf.AdvanceKnownHead(mctx, in.GetStreamId(), at) + }, + in.GetHeadOffset(), + ); err != nil { + return nil, err + } + return &streampb.AdvanceConsumerHeadResponse{ + FrontendResponse: &streampb.AdvanceConsumerHeadOutput{}, + }, nil +} + +func (h *handler) PollMessages( + ctx context.Context, + req *streampb.PollMessagesRequest, +) (*streampb.PollMessagesResponse, error) { + in := req.GetFrontendRequest() + ctx = h.withCallerInfo(ctx, req.GetNamespaceId()) + + ref := refForRun(req.GetNamespaceId(), in.GetStreamId(), in.GetRunId()) + from := in.GetFromOffset() + + state, err := chasm.ReadComponent(ctx, ref, (*stream.Stream).Snapshot, struct{}{}) + if err != nil { + return nil, err + } + + // Blocking is only worth it once the reader is genuinely caught up. + if in.GetWaitNewMessages() && from == state.GetHeadOffset() && !state.GetClosed() { + // The window is re-read below, so only the blocking matters here. + if _, err := h.waitForMessages(ctx, ref, from, state); err != nil { + return nil, err + } + } + + wreq := stream.WindowRequest{From: from, MaxMessages: in.GetMaxMessages(), Topics: in.GetTopics()} + w, err := chasm.ReadComponent(ctx, ref, (*stream.Stream).ReadWindow, wreq) + if err != nil { + return nil, err + } + out, err := formatWindow(w, wreq) + if err != nil { + return nil, err + } + return &streampb.PollMessagesResponse{FrontendResponse: out}, nil +} + +// PollWorkflowMessages reads a stream a workflow owns. +// +// Everything that addresses the stream comes from its owner: the shard, the +// frontier, and the collection the log nodes were written under. That is also +// why the log read below needs no special case. An attached stream's nodes are +// written under the owner's shard, which is the shard this call routed to. +func (h *handler) PollWorkflowMessages( + ctx context.Context, + req *streampb.PollWorkflowMessagesRequest, +) (*streampb.PollWorkflowMessagesResponse, error) { + in := req.GetFrontendRequest() + ctx = h.withCallerInfo(ctx, req.GetNamespaceId()) + + ref := workflowRef(req.GetNamespaceId(), in.GetWorkflowId(), in.GetOwnerRunId()) + name := ownedStreamName(in.GetStreamName()) + from := in.GetFromOffset() + + state, err := h.ownedStreamState(ctx, ref, name) + if err != nil { + return nil, err + } + + if in.GetWaitNewMessages() && from == state.GetHeadOffset() && !state.GetClosed() { + if _, err := h.waitForOwnedMessages(ctx, ref, name, from, state); err != nil { + return nil, err + } + } + + wreq := stream.WindowRequest{From: from, MaxMessages: in.GetMaxMessages(), Topics: in.GetTopics()} + w, err := chasm.ReadComponent(ctx, ref, readOwnedWindow, ownedWindowRequest{Name: name, Window: wreq}) + if err != nil { + return nil, err + } + out, err := formatWindow(w, wreq) + if err != nil { + return nil, err + } + return &streampb.PollWorkflowMessagesResponse{FrontendResponse: out}, nil +} + +// readWindow serves a reader's window out of a frontier the caller resolved. +// Standalone and attached streams differ only in where that frontier comes +// from, so nothing past it is aware of the difference. +// formatWindow turns a component read into the wire response. The read happens +// in the component, so the frontier and the bytes it was served with cannot +// disagree. +func formatWindow(w stream.Window, req stream.WindowRequest) (*streampb.PollMessagesOutput, error) { + out := &streampb.PollMessagesOutput{ + NextOffset: req.From, + HeadOffset: w.State.GetHeadOffset(), + Closed: w.State.GetClosed(), + CloseReason: w.State.GetCloseReason(), + } + if req.From == w.State.GetHeadOffset() { + return out, nil + } + + messages, next, err := stream.CollectMessages(w.Blobs, w.Starts, req.From, w.To, w.Limit, req.Topics) + if err != nil { + return nil, err + } + if next < w.To && len(messages) == 0 && len(req.Topics) > 0 { + // A page that filtered everything out still has to advance, or the + // caller loops forever on the same offsets. Limited to a filtered read + // on purpose: for any other reason a page comes back short, moving the + // reader past offsets it was never given would hide the short read. + next = w.To + } + out.Messages = messages + out.NextOffset = next + return out, nil +} + +// ownedWindowRequest names which attached stream to read and what to read. +type ownedWindowRequest struct { + Name string + Window stream.WindowRequest +} + +// readOwnedWindow reads a stream attached to a workflow. +// +// A closed execution can take no more publishes, from its own Workflow Task or +// from anywhere else, so its stream is finished whether or not a producer said +// so. Without that a reader tailing a workflow that ended stays parked forever. +func readOwnedWindow( + wf *chasmworkflow.Workflow, + cctx chasm.Context, + req ownedWindowRequest, +) (stream.Window, error) { + s := wf.OwnedStream(cctx, req.Name) + if s == nil { + // Nothing published yet, which reads as an empty stream so a reader can + // attach before the first append. + return stream.Window{State: &streampb.StreamState{Closed: !cctx.ExecutionInfo().CloseTime.IsZero()}, To: req.Window.From}, nil + } + w, err := s.ReadWindow(cctx, req.Window) + if err != nil { + return stream.Window{}, err + } + if !cctx.ExecutionInfo().CloseTime.IsZero() { + w.State.Closed = true + } + return w, nil +} + +// ownedStreamState snapshots an attached stream through the component that +// owns it. A stream the workflow has not published to yet reads as an empty +// one, so a reader may attach before the first event. +func (h *handler) ownedStreamState( + ctx context.Context, + ref chasm.ComponentRef, + name string, +) (*streampb.StreamState, error) { + state, err := chasm.ReadComponent(ctx, ref, readOwnedStream, name) + if err != nil { + return nil, err + } + return state, nil +} + +// readOwnedStream snapshots an attached stream and reports whether anything +// can still be added to it. +// +// A closed execution can take no more publishes, from its own Workflow Task or +// from anywhere else, so its stream is finished whether or not a producer said +// so. Without this a reader tailing a workflow that ended stays parked forever. +func readOwnedStream( + wf *chasmworkflow.Workflow, + cctx chasm.Context, + name string, +) (*streampb.StreamState, error) { + state, err := wf.OwnedStreamState(cctx, name) + if err != nil { + return nil, err + } + if state == nil { + state = &streampb.StreamState{} + } + if !cctx.ExecutionInfo().CloseTime.IsZero() { + state.Closed = true + } + return state, nil +} + +// waitForMessages blocks until the head passes the reader's offset or the +// stream closes. On the server's long-poll timeout it returns the state it last +// saw, so the caller gets an empty response and polls again rather than an +// error it would have to distinguish from a real failure. +func (h *handler) waitForMessages( + ctx context.Context, + ref chasm.ComponentRef, + from int64, + current *streampb.StreamState, +) (*streampb.StreamState, error) { + pollCtx, cancel := contextutil.WithDeadlineBuffer(ctx, stream.LongPollTimeout, stream.LongPollBuffer) + defer cancel() + + state, _, err := chasm.PollComponent(pollCtx, ref, + func(s *stream.Stream, _ chasm.Context, offset int64) (*streampb.StreamState, bool, error) { + // Monotonic, as PollComponent requires: the head only advances and + // closed never clears. + if !pollSatisfied(s.State, offset) { + return nil, false, nil + } + return common.CloneProto(s.State), true, nil + }, from) + return pollOutcome(pollCtx, ctx, state, err, current) +} + +// waitForOwnedMessages is waitForMessages against a stream reached through its +// owner. The predicate has to re-resolve the stream on every evaluation, +// because what the poll observes is the owning execution. +func (h *handler) waitForOwnedMessages( + ctx context.Context, + ref chasm.ComponentRef, + name string, + from int64, + current *streampb.StreamState, +) (*streampb.StreamState, error) { + pollCtx, cancel := contextutil.WithDeadlineBuffer(ctx, stream.LongPollTimeout, stream.LongPollBuffer) + defer cancel() + + state, _, err := chasm.PollComponent(pollCtx, ref, + func(wf *chasmworkflow.Workflow, cctx chasm.Context, offset int64) (*streampb.StreamState, bool, error) { + owned, err := readOwnedStream(wf, cctx, name) + if err != nil { + return nil, false, err + } + if !pollSatisfied(owned, offset) { + return nil, false, nil + } + return owned, true, nil + }, from) + return pollOutcome(pollCtx, ctx, state, err, current) +} + +// pollSatisfied is the monotonic condition PollComponent requires: the head +// only advances and closed never clears. +func pollSatisfied(state *streampb.StreamState, from int64) bool { + return state.GetHeadOffset() > from || state.GetClosed() +} + +// pollOutcome turns a long-poll result into the state the reader should be +// served. +func pollOutcome( + pollCtx, callerCtx context.Context, + state *streampb.StreamState, + err error, + current *streampb.StreamState, +) (*streampb.StreamState, error) { + if err != nil { + if pollCtx.Err() != nil && callerCtx.Err() == nil { + // Our long-poll budget expired, not the caller's. Hand back the + // state we already had so the reader gets an empty response and + // polls again, rather than an error it has to tell apart from a + // real failure. + return current, nil + } + return nil, err + } + if state == nil { + return current, nil + } + return state, nil +} + +func (h *handler) DescribeStream( + ctx context.Context, + req *streampb.DescribeStreamRequest, +) (*streampb.DescribeStreamResponse, error) { + in := req.GetFrontendRequest() + state, err := chasm.ReadComponent(ctx, + refFor(req.GetNamespaceId(), in.GetStreamId()), (*stream.Stream).Snapshot, struct{}{}) + if err != nil { + return nil, err + } + return &streampb.DescribeStreamResponse{ + FrontendResponse: &streampb.DescribeStreamOutput{State: state}, + }, nil +} + +// DescribeWorkflowStream reports the frontier of a stream a workflow owns. A +// reader needs it to start at the tail rather than at the beginning, which an +// attached stream offers no other way to find. +func (h *handler) DescribeWorkflowStream( + ctx context.Context, + req *streampb.DescribeWorkflowStreamRequest, +) (*streampb.DescribeWorkflowStreamResponse, error) { + in := req.GetFrontendRequest() + ctx = h.withCallerInfo(ctx, req.GetNamespaceId()) + + state, err := h.ownedStreamState(ctx, + workflowRef(req.GetNamespaceId(), in.GetWorkflowId(), in.GetOwnerRunId()), + ownedStreamName(in.GetStreamName())) + if err != nil { + return nil, err + } + return &streampb.DescribeWorkflowStreamResponse{ + FrontendResponse: &streampb.DescribeStreamOutput{State: state}, + }, nil +} + +func (h *handler) CloseStream( + ctx context.Context, + req *streampb.CloseStreamRequest, +) (*streampb.CloseStreamResponse, error) { + in := req.GetFrontendRequest() + _, _, err := chasm.UpdateComponent( + ctx, + refFor(req.GetNamespaceId(), in.GetStreamId()), + func(s *stream.Stream, mctx chasm.MutableContext, reason *commonpb.Payload) (struct{}, error) { + return struct{}{}, s.CloseAndSchedule(mctx, reason) + }, + in.GetReason(), + ) + if err != nil { + return nil, err + } + return &streampb.CloseStreamResponse{FrontendResponse: &streampb.CloseStreamOutput{}}, nil +} + +func (h *handler) TruncateStream( + ctx context.Context, + req *streampb.TruncateStreamRequest, +) (*streampb.TruncateStreamResponse, error) { + in := req.GetFrontendRequest() + + if _, _, err := chasm.UpdateComponent( + ctx, + refFor(req.GetNamespaceId(), in.GetStreamId()), + func(s *stream.Stream, mctx chasm.MutableContext, newBase int64) (struct{}, error) { + return struct{}{}, s.Truncate(mctx, newBase) + }, + in.GetNewBaseOffset(), + ); err != nil { + return nil, err + } + return &streampb.TruncateStreamResponse{FrontendResponse: &streampb.TruncateStreamOutput{}}, nil +} + +// ListStreams is intentionally not implemented here. It queries visibility, so +// it has no business ID to route on and the frontend answers it directly. + +func (h *handler) DeleteStream( + ctx context.Context, + req *streampb.DeleteStreamRequest, +) (*streampb.DeleteStreamResponse, error) { + in := req.GetFrontendRequest() + key := chasm.ExecutionKey{NamespaceID: req.GetNamespaceId(), BusinessID: in.GetStreamId()} + + // The payload is component state, so deleting the execution takes it too. + if err := chasm.DeleteExecution[*stream.Stream](ctx, key, chasm.DeleteExecutionRequest{}); err != nil { + return nil, err + } + return &streampb.DeleteStreamResponse{FrontendResponse: &streampb.DeleteStreamOutput{}}, nil +} diff --git a/chasm/lib/stream/service/library.go b/chasm/lib/stream/service/library.go new file mode 100644 index 00000000000..ffc6bb9a220 --- /dev/null +++ b/chasm/lib/stream/service/library.go @@ -0,0 +1,85 @@ +package service + +import ( + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "google.golang.org/grpc" +) + +const ( + libraryName = "stream" + componentName = "stream" +) + +var ( + Archetype = chasm.FullyQualifiedName(libraryName, componentName) + ArchetypeID = chasm.GenerateTypeID(Archetype) +) + +type library struct { + chasm.UnimplementedLibrary + handler *handler + retention *retentionTaskHandler + notifyConsumers *notifyConsumersTaskHandler +} + +func newLibrary(h *handler, retention *retentionTaskHandler, notifyConsumers *notifyConsumersTaskHandler) *library { + return &library{handler: h, retention: retention, notifyConsumers: notifyConsumers} +} + +// componentOnlyLibrary registers the component without the service, which is +// what the frontend needs in order to serialize component references. +type componentOnlyLibrary struct { + chasm.UnimplementedLibrary +} + +func newComponentOnlyLibrary() *componentOnlyLibrary { + return &componentOnlyLibrary{} +} + +func (l *componentOnlyLibrary) Name() string { + return libraryName +} + +func (l *componentOnlyLibrary) Components() []*chasm.RegistrableComponent { + return components() +} + +func components() []*chasm.RegistrableComponent { + return []*chasm.RegistrableComponent{ + chasm.NewRegistrableComponent[*stream.Stream]( + componentName, + chasm.WithBusinessIDAlias("StreamId"), + ), + // Registered here rather than with the workflow library because it is + // this package's type, even though it only ever hangs off a consuming + // workflow. + chasm.NewRegistrableComponent[*stream.Cursor]("streamCursor"), + } +} + +func (l *library) Name() string { + return libraryName +} + +func (l *library) Components() []*chasm.RegistrableComponent { + return components() +} + +func (l *library) Tasks() []*chasm.RegistrableTask { + return []*chasm.RegistrableTask{ + chasm.NewRegistrableSideEffectTask( + "streamRetention", + l.retention, + ), + chasm.NewRegistrableSideEffectTask( + "streamNotifyConsumers", + l.notifyConsumers, + ), + } +} + +func (l *library) RegisterServices(server *grpc.Server) { + streampb.RegisterStreamServiceServer(server, l.handler) +} diff --git a/chasm/lib/stream/service/tasks.go b/chasm/lib/stream/service/tasks.go new file mode 100644 index 00000000000..9e679e675b9 --- /dev/null +++ b/chasm/lib/stream/service/tasks.go @@ -0,0 +1,209 @@ +package service + +import ( + "context" + "errors" + + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/service/history/shard" +) + +// retentionTaskHandler deletes a stream once its retention has elapsed. Close +// only seals; deletion is deliberately later, so a consumer can still drain a +// finished stream without coordinating a shutdown with the producer. +type retentionTaskHandler struct { + chasm.SideEffectTaskHandlerBase[*streampb.StreamRetentionTask] + + shardController shard.Controller + namespaceRegistry namespace.Registry + logger log.Logger +} + +func newRetentionTaskHandler( + shardController shard.Controller, + namespaceRegistry namespace.Registry, + logger log.Logger, +) *retentionTaskHandler { + return &retentionTaskHandler{ + shardController: shardController, + namespaceRegistry: namespaceRegistry, + logger: logger, + } +} + +func (h *retentionTaskHandler) Validate( + _ chasm.Context, + s *stream.Stream, + _ chasm.TaskInvocation, + _ *streampb.StreamRetentionTask, +) (bool, error) { + // A stream that was reopened, or never closed, has nothing to expire. The + // task is scheduled at close and only meaningful while that still holds. + return s.State.GetClosed(), nil +} + +func (h *retentionTaskHandler) Execute( + ctx context.Context, + ref chasm.ComponentRef, + _ chasm.TaskAttributes, + _ *streampb.StreamRetentionTask, +) error { + namespaceID := ref.NamespaceID + + // Runs outside a request, so nothing has tagged the context yet. Deletions + // still have to be attributed to the namespace they belong to. + if name, nsErr := h.namespaceRegistry.GetNamespaceName(namespace.ID(namespaceID)); nsErr == nil { + ctx = headers.SetCallerInfo(ctx, headers.NewBackgroundLowCallerInfo(name.String())) + } + + // The payload is component state, so deleting the execution takes it too. + return chasm.DeleteExecution[*stream.Stream](ctx, ref.ExecutionKey, chasm.DeleteExecutionRequest{}) +} + +func (h *retentionTaskHandler) Discard( + _ context.Context, + _ chasm.ComponentRef, + _ chasm.TaskAttributes, + _ *streampb.StreamRetentionTask, +) error { + // Nothing to undo: the task carries no side effect until it executes. + return nil +} + +// notifyConsumersTaskHandler tells workflows in other executions that the +// stream moved. +// +// Appending never schedules a workflow task by itself, because a stream item is +// data an execution produced rather than a decision input to it. A workflow +// that subscribed is the exception, and it has no other way to find out: it +// cannot read another execution's frontier while closing its own transaction. +// So the frontier is pushed into its cursor, which dirties that execution and +// lets its own transaction close decide it owes a workflow task. +type notifyConsumersTaskHandler struct { + chasm.SideEffectTaskHandlerBase[*streampb.StreamNotifyConsumersTask] + + namespaceRegistry namespace.Registry + logger log.Logger + + // This task runs on the stream's shard. Its consumers live wherever their + // own executions do, so telling them goes back out through the service to + // be routed rather than resolved here. + routed streampb.StreamServiceClient +} + +func newNotifyConsumersTaskHandler( + namespaceRegistry namespace.Registry, + logger log.Logger, + routed streampb.StreamServiceClient, +) *notifyConsumersTaskHandler { + return ¬ifyConsumersTaskHandler{ + namespaceRegistry: namespaceRegistry, + logger: logger, + routed: routed, + } +} + +func (h *notifyConsumersTaskHandler) Validate( + _ chasm.Context, + s *stream.Stream, + _ chasm.TaskInvocation, + _ *streampb.StreamNotifyConsumersTask, +) (bool, error) { + for _, consumer := range s.State.GetConsumers() { + if consumer.GetExternal() && consumer.GetActive() { + return true, nil + } + } + return false, nil +} + +func (h *notifyConsumersTaskHandler) Execute( + ctx context.Context, + ref chasm.ComponentRef, + _ chasm.TaskAttributes, + _ *streampb.StreamNotifyConsumersTask, +) error { + namespaceID := ref.NamespaceID + streamID := ref.BusinessID + + if name, nsErr := h.namespaceRegistry.GetNamespaceName(namespace.ID(namespaceID)); nsErr == nil { + ctx = headers.SetCallerInfo(ctx, headers.NewBackgroundLowCallerInfo(name.String())) + } + + state, err := chasm.ReadComponent(ctx, ref, (*stream.Stream).Snapshot, struct{}{}) + if err != nil { + return err + } + head := state.GetHeadOffset() + + // Collected rather than returned at the first failure, so one unreachable + // consumer does not stop the others being told in this pass, and returned + // at the end so the task retries rather than dropping the notification. + // + // Dropping it is not survivable for a consumer on another host. The engine + // resolves this ref against the local shard controller, so a consumer whose + // shard lives elsewhere fails every time, its known head never advances, + // and delivery clips to it: Path C across executions never delivers at all. + // A retry does not fix that. It makes it visible, which a warning did not. + var notifyErrs []error + + for consumerID, consumer := range state.GetConsumers() { + if !consumer.GetExternal() || !consumer.GetActive() || consumer.GetOffset() >= head { + continue + } + + _, err := h.routed.AdvanceConsumerHead(ctx, &streampb.AdvanceConsumerHeadRequest{ + NamespaceId: namespaceID, + FrontendRequest: &streampb.AdvanceConsumerHeadInput{ + WorkflowId: consumer.GetWorkflowId(), + OwnerRunId: consumer.GetRunId(), + StreamId: streamID, + HeadOffset: head, + }, + }) + var gone *serviceerror.NotFound + switch { + case errors.As(err, &gone): + // The consumer's execution is gone, so its replay floor is holding + // storage for a recovery that can no longer be asked for. This is + // the one place that finds out: the probe happens exactly when the + // frontier has moved past the consumer, which is exactly when the + // floor starts to matter. + if _, _, releaseErr := chasm.UpdateComponent( + ctx, ref, + func(s *stream.Stream, mctx chasm.MutableContext, id string) (struct{}, error) { + s.DeregisterConsumer(mctx, id) + return struct{}{}, nil + }, + consumerID, + ); releaseErr != nil { + notifyErrs = append(notifyErrs, releaseErr) + } + case err != nil: + h.logger.Error("failed to tell a stream consumer that the frontier moved", + tag.NewStringTag("stream-id", streamID), + tag.NewStringTag("consumer-workflow-id", consumer.GetWorkflowId()), + tag.Error(err)) + notifyErrs = append(notifyErrs, err) + default: + // Told, and still there. Nothing to clean up. + } + } + return errors.Join(notifyErrs...) +} + +func (h *notifyConsumersTaskHandler) Discard( + _ context.Context, + _ chasm.ComponentRef, + _ chasm.TaskAttributes, + _ *streampb.StreamNotifyConsumersTask, +) error { + return nil +} diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go new file mode 100644 index 00000000000..3735530e3a2 --- /dev/null +++ b/chasm/lib/stream/stream.go @@ -0,0 +1,695 @@ +package stream + +import ( + "crypto/sha256" + "maps" + "slices" + "time" + + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "go.temporal.io/server/common" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Stream is a durable, offset-addressed append-only sequence. It holds only the +// frontier: the payload bytes live in the log (see log.go), so this state is +// O(producers + consumers) no matter how long the stream gets. +// +// Appending never schedules a workflow task. A stream item is data produced by +// an execution, not a decision input to it, so nothing in a workflow's state +// machine advances because one arrived. +type Stream struct { + chasm.UnimplementedComponent + + State *streampb.StreamState + + // Batches holds the payload, each keyed by the offset it starts at. They are + // data nodes, so they replicate with the component and are reclaimed with + // it, and a retry addresses the same key rather than racing it. + // + // They also live in mutable state, which caps a stream at the execution + // size limit. That is the trade for not owning a store. + Batches chasm.Map[int64, *commonpb.DataBlob] + + // Present so streams are listable. Operators need to find them the same way + // they find workflows, and without this the only way to reach a stream is + // to already know its ID. + Visibility chasm.Field[*chasm.Visibility] +} + +type NewStreamRequest struct { + CollectionID string + BucketSize int64 + Lifecycle *streampb.StreamLifecycle + + // Attached means the stream is a subcomponent of another execution rather + // than a root. CHASM requires a visibility component to be an immediate + // child of the root, so an attached stream carries none and is found + // through its owner instead of through ListStreams. + Attached bool +} + +type AddMessagesRequest struct { + Messages []*streampb.StreamMessage + + // Optional idempotency. A producer supplies either an identity and + // sequence, or an expected offset, or neither and accepts at-least-once. + ProducerID string + Sequence int64 + ExpectedOffset *int64 + + // Optional fencing. Rejected if below the stream's current epoch. + OwnerEpoch int64 +} + +type AddMessagesResult struct { + FirstOffset int64 + NextOffset int64 + Count int64 + + // True when a retry matched a recorded producer sequence, so nothing was + // appended and the original offsets are returned. + Deduplicated bool + + // The bytes this append wrote, so a caller can prime a cache without + // reading them back. Nil when deduplicated. + Blob *commonpb.DataBlob +} + +func NewStream(ctx chasm.MutableContext, req NewStreamRequest) (*Stream, error) { + bucketSize := req.BucketSize + if bucketSize <= 0 { + bucketSize = DefaultBucketSize + } + visibility := chasm.NewEmptyField[*chasm.Visibility]() + if !req.Attached { + visibility = chasm.NewComponentField(ctx, chasm.NewVisibility(ctx)) + } + return &Stream{ + Visibility: visibility, + Batches: make(chasm.Map[int64, *commonpb.DataBlob]), + State: &streampb.StreamState{ + CollectionId: req.CollectionID, + BucketSize: bucketSize, + Lifecycle: req.Lifecycle, + Producers: make(map[string]*streampb.ProducerCursor), + Consumers: make(map[string]*streampb.ConsumerCursor), + }, + }, nil +} + +// ContextMetadata satisfies chasm.RootComponent. A stream carries no metadata +// worth propagating to the request context. +func (s *Stream) ContextMetadata(_ chasm.Context) map[string]string { + return nil +} + +// Terminate seals the stream so a forced shutdown does not leave it accepting +// writes. Data already appended stays readable, because consumers may have read +// it and the stream is append-only. +func (s *Stream) Terminate( + mctx chasm.MutableContext, + req chasm.TerminateComponentRequest, +) (chasm.TerminateComponentResponse, error) { + reason := &commonpb.Payload{Data: []byte(req.Reason)} + return chasm.TerminateComponentResponse{}, s.CloseAndSchedule(mctx, reason) +} + +// Snapshot returns a copy of the frontier for read paths. It is a copy because +// the caller reads it outside the transition that produced it. +func (s *Stream) Snapshot(_ chasm.Context, _ struct{}) (*streampb.StreamState, error) { + return common.CloneProto(s.State), nil +} + +func (s *Stream) LifecycleState(_ chasm.Context) chasm.LifecycleState { + if s.State.Closed { + return chasm.LifecycleStateCompleted + } + return chasm.LifecycleStateRunning +} + +// AddMessages assigns a contiguous offset range and writes the bytes into the +// component, so the payload and the frontier commit in one transaction. A torn +// append is therefore not a state anyone can observe. +func (s *Stream) AddMessages( + mctx chasm.MutableContext, + req AddMessagesRequest, +) (AddMessagesResult, error) { + if s.State.Closed { + return AddMessagesResult{}, serviceerror.NewFailedPrecondition("stream is closed") + } + if len(req.Messages) == 0 { + return AddMessagesResult{}, serviceerror.NewInvalidArgument("no messages to append") + } + if len(req.Messages) > MaxMessagesPerBatch { + return AddMessagesResult{}, serviceerror.NewInvalidArgumentf( + "batch of %d exceeds the limit of %d messages", len(req.Messages), MaxMessagesPerBatch) + } + if err := checkBatchBytes(req.Messages); err != nil { + return AddMessagesResult{}, err + } + + blob, err := marshalBatch(req.Messages) + if err != nil { + return AddMessagesResult{}, err + } + hash := contentHash(blob.Data) + + if replay, err := s.checkProducer(req, hash); err != nil || replay != nil { + if err != nil { + return AddMessagesResult{}, err + } + return *replay, nil + } + + if req.OwnerEpoch != 0 && req.OwnerEpoch < s.State.OwnerEpoch { + return AddMessagesResult{}, serviceerror.NewFailedPrecondition("producer has been fenced") + } + + // After the retry check, so a known producer is never rejected for room. + if err := s.checkProducerRoom(req.ProducerID); err != nil { + return AddMessagesResult{}, err + } + + // Before anything is written, because the alternative is to write and then + // discover the cap can only be met by deleting bytes a consumer's committed + // History still refers to. Refusing the write is the honest half of that + // choice: capacity may constrain what is admitted, and may not quietly take + // back a workflow's ability to replay a decision it already made. + if err := s.checkCapRoom(int64(len(req.Messages))); err != nil { + return AddMessagesResult{}, err + } + + if req.ExpectedOffset != nil && *req.ExpectedOffset != s.State.HeadOffset { + return AddMessagesResult{}, serviceerror.NewAlreadyExistsf( + "expected offset %d but stream head is %d", *req.ExpectedOffset, s.State.HeadOffset) + } + + first := s.State.HeadOffset + count := int64(len(req.Messages)) + + if s.Batches == nil { + s.Batches = make(chasm.Map[int64, *commonpb.DataBlob]) + } + s.Batches[first] = chasm.NewDataField(mctx, blob) + + s.State.HeadOffset = first + count + if req.ProducerID != "" { + if s.State.Producers == nil { + s.State.Producers = make(map[string]*streampb.ProducerCursor) + } + s.State.Producers[req.ProducerID] = &streampb.ProducerCursor{ + Seq: req.Sequence, + FirstOffset: first, + Count: count, + ContentHash: hash, + } + } + + s.applyCap() + result := AddMessagesResult{ + FirstOffset: first, + NextOffset: s.State.HeadOffset, + Count: count, + Blob: blob, + } + s.notifyConsumers(mctx) + return result, nil +} + +// notifyConsumers schedules the wake for consumers this append left behind. +// +// Only a workflow in another execution needs it. One consuming a stream it owns +// sees the new frontier while closing its own transaction, so waking it through +// a task would only duplicate a decision already made locally. +func (s *Stream) notifyConsumers(mctx chasm.MutableContext) { + // The append path previews itself against a detached copy to work out which + // log node to write, and that preview has no transition to attach a task + // to. Only the real transition, which carries a context, schedules one. + if mctx == nil { + return + } + for _, consumer := range s.State.Consumers { + if consumer.GetExternal() && consumer.GetActive() && consumer.GetOffset() < s.State.HeadOffset { + mctx.AddTask(s, chasm.TaskAttributes{ScheduledTime: mctx.Now(s)}, &streampb.StreamNotifyConsumersTask{}) + return + } + } +} + +// checkProducerRoom keeps the dedup table bounded. +// +// Entries whose whole batch sits below the floor go first: a retry of a batch +// that truncation already removed cannot be served its recorded offsets +// anyway, so the entry has no use left. +func (s *Stream) checkProducerRoom(producerID string) error { + if producerID == "" { + return nil + } + if _, known := s.State.Producers[producerID]; known { + return nil + } + if len(s.State.Producers) < MaxProducersPerStream { + return nil + } + for id, cursor := range s.State.Producers { + if cursor.GetFirstOffset()+cursor.GetCount() <= s.State.GetBaseOffset() { + delete(s.State.Producers, id) + } + } + if len(s.State.Producers) >= MaxProducersPerStream { + return serviceerror.NewInvalidArgumentf( + "stream already tracks %d producers, which is the limit", MaxProducersPerStream) + } + return nil +} + +// checkProducer applies per-producer idempotency. It returns a replay result +// when the request is a genuine retry, and an error when it is not a retry but +// cannot be accepted either. +func (s *Stream) checkProducer(req AddMessagesRequest, hash []byte) (*AddMessagesResult, error) { + if req.ProducerID == "" { + return nil, nil + } + cursor := s.State.Producers[req.ProducerID] + if cursor == nil { + return nil, nil + } + if cursor.Fenced { + return nil, serviceerror.NewFailedPrecondition("producer has finished writing to this stream") + } + if req.Sequence > cursor.Seq { + return nil, nil + } + if req.Sequence < cursor.Seq { + return nil, serviceerror.NewInvalidArgumentf( + "stale producer sequence %d, last accepted is %d", req.Sequence, cursor.Seq) + } + // Same sequence. Identical content is a retry; different content is a + // client bug, and returning the recorded offsets would report success while + // silently dropping the caller's data. + if !equalHash(cursor.ContentHash, hash) { + return nil, serviceerror.NewInvalidArgumentf( + "producer sequence %d already used with different content", req.Sequence) + } + return &AddMessagesResult{ + FirstOffset: cursor.FirstOffset, + NextOffset: cursor.FirstOffset + cursor.Count, + Count: cursor.Count, + Deduplicated: true, + }, nil +} + +// FinishWriting ends one producer's writes without closing the stream, so other +// producers carry on. Weaker than Close on purpose. +func (s *Stream) FinishWriting(_ chasm.MutableContext, producerID string) error { + if producerID == "" { + return serviceerror.NewInvalidArgument("producer id is required") + } + if s.State.Producers == nil { + s.State.Producers = make(map[string]*streampb.ProducerCursor) + } + cursor := s.State.Producers[producerID] + if cursor == nil { + cursor = &streampb.ProducerCursor{Seq: -1} + s.State.Producers[producerID] = cursor + } + cursor.Fenced = true + return nil +} + +// Close seals the stream. It does not delete it: a closed stream stays readable +// through retention, which is what removes the shutdown handshake the current +// signal-based implementation forces on users. +// Close returns when retention deletion should be scheduled, or the zero time +// if the stream was already closed or has no retention configured. Scheduling +// is the caller's job, which keeps the component a pure state transition and +// testable without a live context. +func (s *Stream) Close(now time.Time, reason *commonpb.Payload) time.Time { + if s.State.Closed { + return time.Time{} + } + s.State.Closed = true + s.State.CloseReason = reason + s.State.CloseTime = timestamppb.New(now) + + retention := s.State.GetLifecycle().GetRetention().AsDuration() + if retention <= 0 { + return time.Time{} + } + return now.Add(retention) +} + +// CloseAndSchedule closes the stream and arms retention if it asked for it. +func (s *Stream) CloseAndSchedule(mctx chasm.MutableContext, reason *commonpb.Payload) error { + if at := s.Close(mctx.Now(s), reason); !at.IsZero() { + mctx.AddTask(s, chasm.TaskAttributes{ScheduledTime: at}, &streampb.StreamRetentionTask{}) + } + return nil +} + +// Truncate advances the readable floor. +// +// It stops at an active consumer's replay floor. A workflow that consumed a +// range recorded that range in its History and can be asked to replay from it, +// so those bytes are part of its recovery rather than spare capacity. Dropping +// them succeeds here and fails much later, during a replay nobody is watching, +// which is the worst place to find out. +// +// The floor is released by deregistering the consumer, which is an act someone +// takes deliberately. An operator who means to drop the bytes anyway does that +// first, and then this call goes through. +// +// A consumer that is behind but not active is not protected. Reading from +// below the base is an error naming where the stream now starts, the same +// answer a log with a retention window gives anywhere else. +func (s *Stream) Truncate(_ chasm.MutableContext, newBase int64) error { + if newBase < s.State.BaseOffset { + return serviceerror.NewInvalidArgumentf( + "cannot truncate backwards from %d to %d", s.State.BaseOffset, newBase) + } + if newBase > s.State.HeadOffset { + return serviceerror.NewInvalidArgumentf( + "cannot truncate past head offset %d", s.State.HeadOffset) + } + if floor, holder, pinned := s.replayFloor(); pinned && newBase > floor { + return serviceerror.NewFailedPreconditionf( + "cannot truncate to %d: consumer %q still depends on offset %d and above "+ + "to replay; deregister it first if those messages are no longer needed", + newBase, holder, floor) + } + s.State.BaseOffset = newBase + s.reclaim(newBase) + return nil +} + +// replayFloor is the oldest offset any active consumer's History still depends +// on, and who is holding it. Named, because a refusal that does not say which +// consumer to look at leaves the operator with nothing to act on. +func (s *Stream) replayFloor() (int64, string, bool) { + var floor int64 + var holder string + found := false + for id, c := range s.State.Consumers { + if !c.GetActive() { + continue + } + if !found || c.GetReplayFloor() < floor { + floor = c.GetReplayFloor() + holder = id + found = true + } + } + return floor, holder, found +} + +// reclaim drops batches lying entirely below the readable floor. A batch +// straddling the floor stays, because the offsets above it are still readable. +func (s *Stream) reclaim(newBase int64) { + starts := s.batchStarts() + for i, start := range starts { + end := s.State.HeadOffset + if i+1 < len(starts) { + end = starts[i+1] + } + if end > newBase { + return + } + delete(s.Batches, start) + } +} + +// batchStarts returns the batch keys in offset order. Reading and reclaiming +// both need where a batch ends, which is where the next one begins. +func (s *Stream) batchStarts() []int64 { + return slices.Sorted(maps.Keys(s.Batches)) +} + +// WindowRequest asks for whatever a reader can be given from an offset. +type WindowRequest struct { + From int64 + MaxMessages int32 + Topics []string +} + +// Window is one read's worth: the frontier it was served against, the batches +// covering the range, and the range itself. +type Window struct { + State *streampb.StreamState + Blobs []*commonpb.DataBlob + Starts []int64 + To int64 + Limit int +} + +// ReadWindow serves a read from the component, so the frontier and the bytes +// come from one view. Read separately they can disagree, because the frontier +// moves while the bytes are being fetched. +func (s *Stream) ReadWindow(ctx chasm.Context, req WindowRequest) (Window, error) { + if req.From < s.State.BaseOffset { + return Window{}, serviceerror.NewFailedPreconditionf( + "offset %d has been truncated, the stream starts at %d", req.From, s.State.BaseOffset) + } + if req.From > s.State.HeadOffset { + return Window{}, serviceerror.NewInvalidArgumentf( + "offset %d is past the stream head %d", req.From, s.State.HeadOffset) + } + + // Clamped at both ends. The caller picks the page size, and an unclamped + // one lets a single poll ask the store to materialise the whole stream. + limit := int(req.MaxMessages) + if limit <= 0 || limit > DefaultMaxMessagesPerPoll { + limit = DefaultMaxMessagesPerPoll + } + w := Window{State: common.CloneProto(s.State), To: req.From, Limit: limit} + if req.From == s.State.HeadOffset { + return w, nil + } + + // Clip to what the caller can actually be given. One offset is one message, + // so the bound is exact. Without it a poll for a single message off a long + // stream materialises every batch to the head before trimming. + w.To = min(s.State.HeadOffset, req.From+int64(limit)) + blobs, starts, err := s.ReadBatches(ctx, req.From, w.To, 0) + if err != nil { + return Window{}, err + } + w.Blobs, w.Starts = blobs, starts + return w, nil +} + +// ReadBatches returns the batches covering [from, to), oldest first, alongside +// the offset each one starts at. A read landing mid-batch gets the batch +// holding it, because a consumer asks for an offset rather than for a batch. +// Blobs come back unparsed: decoding user payloads is the SDK's job. +func (s *Stream) ReadBatches( + ctx chasm.Context, + from int64, + to int64, + maxBatches int, +) ([]*commonpb.DataBlob, []int64, error) { + if from >= to { + return nil, nil, nil + } + var blobs []*commonpb.DataBlob + var starts []int64 + all := s.batchStarts() + for i, start := range all { + end := s.State.HeadOffset + if i+1 < len(all) { + end = all[i+1] + } + if end <= from { + continue + } + if start >= to { + break + } + blobs = append(blobs, s.Batches[start].Get(ctx)) + starts = append(starts, start) + if maxBatches > 0 && len(blobs) >= maxBatches { + break + } + } + return blobs, starts, nil +} + +// applyCap advances the readable floor when the stream is over its message cap. +// Evaluated at the end of a successful append rather than by a sweeper: the +// append transition is already writing, so folding the check into it costs +// nothing and keeps the cap tight instead of eventually true. +func (s *Stream) applyCap() { + maxItems := s.State.GetLifecycle().GetMaxItems() + if maxItems <= 0 { + return + } + readable := s.State.HeadOffset - s.State.BaseOffset + if readable <= maxItems { + return + } + newBase := s.State.HeadOffset - maxItems + // Clamped rather than refused, because refusing belongs to admission and + // has already happened: checkCapRoom turned away the append that would have + // needed this. Reaching the clamp means a consumer registered after the + // messages were written, and keeping its bytes is still the right answer. + if floor, _, pinned := s.replayFloor(); pinned && newBase > floor { + newBase = floor + } + if newBase <= s.State.BaseOffset { + return + } + s.State.BaseOffset = newBase + s.reclaim(newBase) +} + +// checkCapRoom refuses an append the cap could only absorb by dropping bytes an +// active consumer still needs. +// +// A capped stream with no consumer behaves as before: the oldest messages go. +// The refusal only arrives when honouring the cap and honouring a recorded +// consumption are the same messages, and it names the consumer so the operator +// knows what to do about it. +func (s *Stream) checkCapRoom(count int64) error { + maxItems := s.State.GetLifecycle().GetMaxItems() + if maxItems <= 0 { + return nil + } + wantBase := s.State.HeadOffset + count - maxItems + if wantBase <= s.State.BaseOffset { + return nil + } + floor, holder, pinned := s.replayFloor() + if !pinned || wantBase <= floor { + return nil + } + return serviceerror.NewResourceExhaustedf( + enumspb.RESOURCE_EXHAUSTED_CAUSE_UNSPECIFIED, + "stream is at its cap of %d messages and consumer %q still depends on "+ + "offset %d and above to replay; the append would have to delete those "+ + "messages to make room", + maxItems, holder, floor) +} + +// RegisterConsumer records an in-workflow consumer, so appends know who to wake +// and retention knows what it may not delete. +// +// The floor it records is where the subscription started, not where it has read +// to. The ranges this consumer already took are written into its History, and a +// replay is asked to reproduce them, so the bytes behind the read position are +// the ones a recovery needs. Deregistering releases the floor. +func (s *Stream) RegisterConsumer( + _ chasm.MutableContext, + consumerID string, + workflowID string, + runID string, + offset int64, + external bool, +) error { + if consumerID == "" { + return serviceerror.NewInvalidArgument("consumer id is required") + } + if offset < s.State.BaseOffset { + return serviceerror.NewFailedPreconditionf( + "offset %d is below the stream's floor of %d", offset, s.State.BaseOffset) + } + if _, known := s.State.Consumers[consumerID]; !known && + len(s.State.Consumers) >= MaxConsumersPerStream { + return serviceerror.NewInvalidArgumentf( + "stream already has %d consumers, which is the limit", MaxConsumersPerStream) + } + if s.State.Consumers == nil { + s.State.Consumers = make(map[string]*streampb.ConsumerCursor) + } + if existing, ok := s.State.Consumers[consumerID]; ok { + // A consumer coming back after its floor was released can find the + // stream has moved past what its History refers to. Saying so here is + // the only chance to say it before the workflow depends on it again. + if existing.GetReplayFloor() < s.State.BaseOffset { + return serviceerror.NewFailedPreconditionf( + "consumer %q recorded offset %d, and the stream now starts at %d", + consumerID, existing.GetReplayFloor(), s.State.BaseOffset) + } + existing.Active = true + return nil + } + s.State.Consumers[consumerID] = &streampb.ConsumerCursor{ + WorkflowId: workflowID, + RunId: runID, + Offset: offset, + Active: true, + External: external, + ReplayFloor: offset, + } + return nil +} + +// AdvanceConsumer moves a consumer's pin forward as it reads. It never moves +// backwards: the floor is what lets a recorded range still be re-read, so +// lowering it would give back a guarantee already written to History. +func (s *Stream) AdvanceConsumer(_ chasm.MutableContext, consumerID string, offset int64) { + consumer, ok := s.State.Consumers[consumerID] + if !ok || offset <= consumer.Offset { + return + } + consumer.Offset = offset +} + +// consumerPin is the lowest offset any active consumer has reached. Nothing in +// the write path consults it, by the decision above; it is how the consumer +// table is read back, and what asserts that a re-registration cannot rewind a +// consumer's recorded position. +func (s *Stream) consumerPin() (int64, bool) { + var pin int64 + found := false + for _, c := range s.State.Consumers { + if !c.Active { + continue + } + if !found || c.Offset < pin { + pin = c.Offset + found = true + } + } + return pin, found +} + +// DeregisterConsumer releases the floor a consumer was holding, so retention +// and the message cap can reach its messages again. +func (s *Stream) DeregisterConsumer(_ chasm.MutableContext, consumerID string) { + if consumer, ok := s.State.Consumers[consumerID]; ok { + consumer.Active = false + } +} + +func marshalBatch(messages []*streampb.StreamMessage) (*commonpb.DataBlob, error) { + data, err := proto.Marshal(&streampb.StreamMessageBatch{Messages: messages}) + if err != nil { + return nil, err + } + return &commonpb.DataBlob{ + EncodingType: enumspb.ENCODING_TYPE_PROTO3, + Data: data, + }, nil +} + +func contentHash(data []byte) []byte { + sum := sha256.Sum256(data) + return sum[:] +} + +func equalHash(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go new file mode 100644 index 00000000000..a527fc7002b --- /dev/null +++ b/chasm/lib/stream/stream_test.go @@ -0,0 +1,497 @@ +package stream + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "google.golang.org/protobuf/types/known/durationpb" +) + +// Built directly rather than through NewStream: these exercise state +// transitions, and NewStream also wires a visibility field that needs a live +// context. Construction through the real path is covered end to end in +// tests/stream_test.go. +func newTestStream(t *testing.T, bucketSize int64) *Stream { + t.Helper() + return &Stream{ + State: &streampb.StreamState{ + CollectionId: "col-1", + BucketSize: bucketSize, + Producers: make(map[string]*streampb.ProducerCursor), + Consumers: make(map[string]*streampb.ConsumerCursor), + }, + } +} + +func msgs(bodies ...string) []*streampb.StreamMessage { + out := make([]*streampb.StreamMessage, len(bodies)) + for i, b := range bodies { + out[i] = &streampb.StreamMessage{ + Body: &commonpb.Payload{Data: []byte(b)}, + Kind: streampb.STREAM_MESSAGE_KIND_DATA, + } + } + return out +} + +func TestAddMessagesAssignsContiguousOffsets(t *testing.T) { + s := newTestStream(t, 100) + + first, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c")}) + require.NoError(t, err) + require.Equal(t, int64(0), first.FirstOffset) + require.Equal(t, int64(3), first.Count) + require.Equal(t, int64(3), first.NextOffset) + + second, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("d", "e")}) + require.NoError(t, err) + require.Equal(t, int64(3), second.FirstOffset) + require.Equal(t, int64(5), second.NextOffset) + require.Equal(t, int64(5), s.State.HeadOffset) +} + +func TestAddMessagesWritesTheBatchIntoTheComponent(t *testing.T) { + s := newTestStream(t, 100) + + res, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b")}) + require.NoError(t, err) + require.NotEmpty(t, res.Blob.Data) + + // Keyed by the offset it starts at, which is the key a retry of this + // append writes under, so the retry replaces rather than races. + require.Len(t, s.Batches, 1) + _, ok := s.Batches[0] + require.True(t, ok, "the batch must be keyed by its first offset") + require.Equal(t, int64(2), s.State.HeadOffset) +} + +func TestDedupReturnsOriginalOffsets(t *testing.T) { + s := newTestStream(t, 100) + req := AddMessagesRequest{Messages: msgs("a", "b"), ProducerID: "p1", Sequence: 1} + + first, err := s.AddMessages(nil, req) + require.NoError(t, err) + require.False(t, first.Deduplicated) + + retry := req + again, err := s.AddMessages(nil, retry) + require.NoError(t, err) + require.True(t, again.Deduplicated) + require.Equal(t, first.FirstOffset, again.FirstOffset) + require.Nil(t, again.Blob, "a deduplicated retry writes nothing") + require.Equal(t, int64(2), s.State.HeadOffset, "a retry must not advance the head") +} + +func TestDedupRejectsDifferentContent(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("a"), ProducerID: "p1", Sequence: 1, + }) + require.NoError(t, err) + + // Returning the recorded offsets here would report success while dropping + // the caller's data, which is worse than failing. + _, err = s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("different"), ProducerID: "p1", Sequence: 1, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "different content") +} + +func TestExpectedOffsetMismatchReportsHead(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a")}) + require.NoError(t, err) + + stale := int64(0) + _, err = s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("b"), ExpectedOffset: &stale, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "stream head is 1") +} + +func TestOwnerEpochFencesStaleProducer(t *testing.T) { + s := newTestStream(t, 100) + s.State.OwnerEpoch = 5 + + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a"), OwnerEpoch: 4}) + require.Error(t, err) + + _, err = s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a"), OwnerEpoch: 5}) + require.NoError(t, err) +} + +func TestFinishWritingFencesOneProducerOnly(t *testing.T) { + s := newTestStream(t, 100) + require.NoError(t, s.FinishWriting(nil, "p1")) + + _, err := s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("a"), ProducerID: "p1", Sequence: 1, + }) + require.Error(t, err) + + // Another producer is unaffected: finishing is per-producer, not a close. + _, err = s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("a"), ProducerID: "p2", Sequence: 1, + }) + require.NoError(t, err) + require.False(t, s.State.Closed) +} + +func TestCloseRejectsFurtherAppends(t *testing.T) { + s := newTestStream(t, 100) + s.Close(time.Now(), nil) + + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a")}) + require.Error(t, err) + var precondition *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &precondition) +} + +func TestReadSpansBatchesAndStartsAtTheBatchHoldingTheOffset(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c")}) + require.NoError(t, err) + _, err = s.AddMessages(nil, AddMessagesRequest{Messages: msgs("d", "e")}) + require.NoError(t, err) + require.Len(t, s.Batches, 2) + + // A read from offset 1 lands inside the first batch. It gets that batch + // whole, because a consumer asks for an offset and not for a batch, and + // the batch is the smallest thing stored. + blobs, starts, err := s.ReadBatches(nil, 1, 5, 0) + require.NoError(t, err) + require.Len(t, blobs, 2) + require.Equal(t, []int64{0, 3}, starts) + + // A read wholly inside the second batch does not drag the first along. + blobs, starts, err = s.ReadBatches(nil, 3, 5, 0) + require.NoError(t, err) + require.Len(t, blobs, 1) + require.Equal(t, []int64{3}, starts) +} + +func TestReclaimDropsOnlyBatchesFullyBelowTheFloor(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c")}) + require.NoError(t, err) + _, err = s.AddMessages(nil, AddMessagesRequest{Messages: msgs("d", "e")}) + require.NoError(t, err) + + // The floor lands mid-batch, so that batch stays: offsets above the floor + // are still readable and they live in it. + require.NoError(t, s.Truncate(nil, 1)) + require.Len(t, s.Batches, 2) + + // Now the whole first batch is below the floor and can go. + require.NoError(t, s.Truncate(nil, 3)) + require.Len(t, s.Batches, 1) + _, ok := s.Batches[3] + require.True(t, ok, "the batch holding readable offsets must survive") +} + +func TestTruncateStopsAtAnActiveConsumersReplayFloor(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + + require.NoError(t, s.RegisterConsumer(nil, "wf-1", "wf-1", "run-1", 0, false)) + s.AdvanceConsumer(nil, "wf-1", 2) + + // Reading to 2 is exactly what makes offsets 0 and 1 matter: they are in + // this consumer's History and a replay is asked to reproduce them. + err = s.Truncate(nil, 3) + require.ErrorContains(t, err, "still depends on offset 0") + require.Equal(t, int64(0), s.State.BaseOffset) + + // Whatever is above the floor is still spare capacity. + require.NoError(t, s.Truncate(nil, 0)) +} + +func TestTruncateBounds(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b")}) + require.NoError(t, err) + + err = s.Truncate(nil, 1) + require.NoError(t, err) + err = s.Truncate(nil, 0) + require.Error(t, err, "truncation must not go backwards") + err = s.Truncate(nil, 3) + require.Error(t, err, "truncation must not pass the head") +} + +func TestBucketArithmetic(t *testing.T) { + require.Equal(t, int64(0), BucketOf(0, 10)) + require.Equal(t, int64(0), BucketOf(9, 10)) + require.Equal(t, int64(1), BucketOf(10, 10)) + + // Node ids are bucket-relative and start at 1, because the store rejects 0. + require.Equal(t, int64(20), BucketStart(2, 10)) +} + +func TestReclaimableBuckets(t *testing.T) { + // Only buckets lying entirely below the floor are reclaimable, so nothing a + // reader can still ask for is ever dropped. + require.Empty(t, ReclaimableBuckets(0, 3, 4)) + require.Equal(t, []int64{0}, ReclaimableBuckets(0, 4, 4)) + require.Equal(t, []int64{0, 1}, ReclaimableBuckets(0, 8, 4)) + require.Equal(t, []int64{1}, ReclaimableBuckets(4, 8, 4)) + require.Empty(t, ReclaimableBuckets(4, 5, 4)) +} + +func TestCapTruncatesInline(t *testing.T) { + s := newTestStream(t, 4) + s.State.Lifecycle = &streampb.StreamLifecycle{MaxItems: 4} + + for range 4 { + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b")}) + require.NoError(t, err) + } + + // Eight appended, four retained, so the floor sits at 4 and bucket 0 is + // entirely below it. + require.Equal(t, int64(8), s.State.HeadOffset) + require.Equal(t, int64(4), s.State.BaseOffset) +} + +func TestCapRefusesAnAppendItCouldOnlyAbsorbByDroppingReadRecords(t *testing.T) { + s := newTestStream(t, 100) + s.State.Lifecycle = &streampb.StreamLifecycle{MaxItems: 2} + require.NoError(t, s.RegisterConsumer(nil, "wf-1", "wf-1", "run-1", 0, false)) + + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.ErrorContains(t, err, "still depends on offset 0") + require.Equal(t, int64(0), s.State.HeadOffset, "a refused append writes nothing") + + // Nothing is stuck. The consumer going away is what makes room, and it is + // something someone does rather than something that happens quietly. + s.DeregisterConsumer(nil, "wf-1") + _, err = s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + require.Equal(t, int64(2), s.State.BaseOffset, "the cap applies once nobody needs the bytes") +} + +func TestCloseSchedulesRetentionOnlyWhenConfigured(t *testing.T) { + now := time.Now() + + plain := newTestStream(t, 100) + require.True(t, plain.Close(now, nil).IsZero(), "no retention configured, nothing to schedule") + + withRetention := newTestStream(t, 100) + withRetention.State.Lifecycle = &streampb.StreamLifecycle{ + Retention: durationpb.New(time.Hour), + } + at := withRetention.Close(now, nil) + require.Equal(t, now.Add(time.Hour), at) + require.NotNil(t, withRetention.State.CloseTime) + + // Closing twice must not re-arm deletion. + require.True(t, withRetention.Close(now, nil).IsZero()) +} + +func TestRegisterConsumerPinsFromWhereItSubscribed(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + + // Subscribing at 2 says nothing about offsets 0 and 1, so those stay + // droppable and everything from 2 up does not. + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 2, false)) + + require.NoError(t, s.Truncate(nil, 2)) + require.Equal(t, int64(2), s.State.BaseOffset) + require.ErrorContains(t, s.Truncate(nil, 3), "still depends on offset 2") +} + +func TestAdvanceConsumerTracksWhereAConsumerHasReached(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 0, false)) + + // The cursor is what decides whether this consumer is worth waking, and + // nothing else now depends on it. + s.AdvanceConsumer(nil, "workflow:output", 3) + require.Equal(t, int64(3), s.State.Consumers["workflow:output"].Offset) +} + +// Lowering the pin would hand back a guarantee already written to History: a +// recorded range has to stay re-readable. +func TestAdvanceConsumerNeverRewinds(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 0, false)) + + s.AdvanceConsumer(nil, "workflow:output", 3) + s.AdvanceConsumer(nil, "workflow:output", 1) + + pin, ok := s.consumerPin() + require.True(t, ok) + require.Equal(t, int64(3), pin) +} + +func TestRegisterConsumerRejectsAnOffsetBelowTheFloor(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + err = s.Truncate(nil, 2) + require.NoError(t, err) + + err = s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 1, false) + require.ErrorContains(t, err, "below the stream's floor") +} + +// Resubscribing reactivates the existing pin rather than resetting it, so a +// consumer cannot rewind its own floor by subscribing again. +func TestRegisterConsumerTwiceKeepsThePin(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 0, false)) + s.AdvanceConsumer(nil, "workflow:output", 3) + + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 0, false)) + + pin, ok := s.consumerPin() + require.True(t, ok) + require.Equal(t, int64(3), pin) +} + +func TestDeregisterConsumerReleasesThePin(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 1, false)) + + s.DeregisterConsumer(nil, "workflow:output") + + err = s.Truncate(nil, 4) + require.NoError(t, err) +} + +func TestMessageCapStillAppliesWithNoConsumerToProtect(t *testing.T) { + s := newTestStream(t, 100) + s.State.Lifecycle = &streampb.StreamLifecycle{MaxItems: 2} + + for range 3 { + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b")}) + require.NoError(t, err) + } + + // The refusal is about a consumer's recovery, so a stream with none behaves + // exactly as a capped log should. + require.Equal(t, int64(6), s.State.HeadOffset) + require.Equal(t, int64(4), s.State.BaseOffset) +} + +// A consumer that arrives after the messages were written cannot make the cap +// retroactively wrong, so the clamp keeps its bytes and the stream sits over +// its cap until it goes away. +func TestCapClampsToAConsumerThatRegisteredLate(t *testing.T) { + s := newTestStream(t, 100) + s.State.Lifecycle = &streampb.StreamLifecycle{MaxItems: 2} + + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b")}) + require.NoError(t, err) + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 0, false)) + s.State.Lifecycle = &streampb.StreamLifecycle{MaxItems: 1} + s.applyCap() + + require.Equal(t, int64(0), s.State.BaseOffset, "the clamp keeps what the consumer needs") +} + +// The reason the pin was taken out in the first place. It must not come back: +// a consumer that finished has to stop holding storage. +func TestAConsumerThatDeregisteredHoldsNothing(t *testing.T) { + s := newTestStream(t, 100) + s.State.Lifecycle = &streampb.StreamLifecycle{MaxItems: 2} + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 0, false)) + s.DeregisterConsumer(nil, "workflow:output") + + for range 3 { + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b")}) + require.NoError(t, err) + } + require.Equal(t, int64(4), s.State.BaseOffset) +} + +// Coming back to a stream that moved past what its History refers to is the +// case the old code discovered during a replay instead. +func TestReregisteringBelowTheFloorIsRefused(t *testing.T) { + s := newTestStream(t, 100) + _, err := s.AddMessages(nil, AddMessagesRequest{Messages: msgs("a", "b", "c", "d")}) + require.NoError(t, err) + require.NoError(t, s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 0, false)) + s.DeregisterConsumer(nil, "workflow:output") + require.NoError(t, s.Truncate(nil, 2)) + + // Resubscribing further along does not repair the gap. What this consumer + // already recorded starts at 0, and offsets 0 and 1 are gone. + err = s.RegisterConsumer(nil, "workflow:output", "wf-1", "run-1", 3, false) + require.ErrorContains(t, err, "the stream now starts at 2") +} + +// A caller sending a fresh producer id per request would otherwise grow the +// component state until no append fits, which leaves the stream unwritable for +// good rather than failing the call that caused it. +func TestStreamProducerTableIsBounded(t *testing.T) { + s := newTestStream(t, 100000) + + for i := range MaxProducersPerStream { + _, err := s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("m"), + ProducerID: fmt.Sprintf("p%d", i), + Sequence: 1, + }) + require.NoError(t, err) + } + + _, err := s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("one too many"), + ProducerID: "p-over", + Sequence: 1, + }) + var invalid *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalid) + + // A producer already tracked keeps working, so the cap cannot wedge the + // producers that filled it. + _, err = s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("still fine"), + ProducerID: "p0", + Sequence: 2, + }) + require.NoError(t, err) + + // An anonymous append is never blocked by the table. + _, err = s.AddMessages(nil, AddMessagesRequest{ + Messages: msgs("anon"), + }) + require.NoError(t, err) +} + +// Each consumer holds a truncation floor, so an unbounded table pins storage as +// well as growing state. +func TestStreamConsumerTableIsBounded(t *testing.T) { + s := newTestStream(t, 100000) + + for i := range MaxConsumersPerStream { + require.NoError(t, s.RegisterConsumer( + nil, fmt.Sprintf("c%d", i), "wf", "run", 0, true)) + } + + err := s.RegisterConsumer(nil, "c-over", "wf", "run", 0, true) + var invalid *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalid) + + // Re-registering an existing consumer is an update, not a new entry. + require.NoError(t, s.RegisterConsumer(nil, "c0", "wf", "run", 0, true)) +} diff --git a/chasm/lib/workflow/fx.go b/chasm/lib/workflow/fx.go index 52730794faa..dbcdedeceb1 100644 --- a/chasm/lib/workflow/fx.go +++ b/chasm/lib/workflow/fx.go @@ -22,6 +22,9 @@ var Module = fx.Module( ); err != nil { return err } + if err := library.registry.Register(&streamLibrary{}); err != nil { + return err + } return chasmRegistry.Register(library) }), ) diff --git a/chasm/lib/workflow/stream_commands.go b/chasm/lib/workflow/stream_commands.go new file mode 100644 index 00000000000..598d46b91e9 --- /dev/null +++ b/chasm/lib/workflow/stream_commands.go @@ -0,0 +1,304 @@ +package workflow + +import ( + commandpb "go.temporal.io/api/command/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + "go.temporal.io/api/serviceerror" + streampb "go.temporal.io/api/stream/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + streamlib "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" +) + +// DefaultStreamName is the stream a command addresses when it names none. +const DefaultStreamName = "output" + +// handleAddStreamMessagesCommand appends to a stream the workflow owns. +// +// The stream is a co-located subcomponent, so its frontier advances as part of +// the workflow task's own commit: no extra transition and no cross-execution +// write. The log bytes cannot be written here, because a command handler runs +// under the state lock with no context to do I/O from, so they are staged and +// flushed before the commit that makes them visible. +// +// The offsets are known here, unlike a subscription's, so the event is written +// here too rather than being staged for the flush. +func handleAddStreamMessagesCommand( + chasmCtx chasm.MutableContext, + wf *Workflow, + validator Validator, + command *commandpb.Command, + opts CommandHandlerOptions, +) error { + attrs := command.GetAddStreamMessagesCommandAttributes() + if attrs == nil { + return serviceerror.NewInvalidArgument("AddStreamMessagesCommandAttributes is not set") + } + if len(attrs.GetMessages()) == 0 { + return serviceerror.NewInvalidArgument("AddStreamMessages command carries no messages") + } + + // The batch becomes one log node, so the whole batch is what has to fit. + // Left unchecked it fails later in the flush, which surfaces as a + // persistence error out of a task the worker will replay and re-issue + // forever, with nothing naming the batch as the cause. + size := 0 + for _, m := range attrs.GetMessages() { + size += m.Size() + } + if !validator.IsValidPayloadSize(size) { + return FailWorkflowTaskError{ + Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_PAYLOADS_TOO_LARGE, + Message: "AddStreamMessagesCommandAttributes.Messages exceeds size limit", + TerminateWorkflow: true, + } + } + + name := attrs.GetStreamId() + if name == "" { + name = DefaultStreamName + } + + s, err := wf.streamNamed(chasmCtx, name) + if err != nil { + return err + } + + result, err := s.AddMessages(chasmCtx, stream.AddMessagesRequest{ + Messages: toLibraryMessages(attrs.GetMessages()), + }) + if err != nil { + return err + } + // Written even when a producer sequence deduplicated the append, because + // the command was still issued and the event is what the replaying worker + // matches it against. It names the original offsets, which is what a + // deduplicated append resolves to. + wf.RecordStreamMessagesAdded( + name, result.FirstOffset, result.Count, opts.WorkflowTaskCompletedEventID) + return nil +} + +// handleSubscribeStreamCommand registers this workflow as a consumer. +// +// A stream the workflow owns is subscribed here and now, because everything the +// cursor needs is already in this execution. One in another execution cannot +// be: its collection id is the stream's run id and its bucket size is its own, +// and finding either means a lookup a command handler cannot do. Those are +// staged and resolved in the flush before commit, the same way log writes are. +func handleSubscribeStreamCommand( + chasmCtx chasm.MutableContext, + wf *Workflow, + _ Validator, + command *commandpb.Command, + opts CommandHandlerOptions, +) error { + attrs := command.GetSubscribeStreamCommandAttributes() + if attrs == nil { + return serviceerror.NewInvalidArgument("SubscribeStreamCommandAttributes is not set") + } + streamID := attrs.GetStreamId() + if streamID == "" { + return serviceerror.NewInvalidArgument("SubscribeStream command names no stream") + } + + // A second subscribe to the same stream registers nothing, but it still + // gets an event. Every SDK matches issued commands against + // command-generated events in order, so a command that produces none puts + // that matching out of step, which is the whole reason this event exists. + _, already := wf.StreamCursors[streamID] + + // Everything is staged, including a stream this workflow owns, so that the + // resolved start offset and the event recording it are produced in one + // place rather than two. + wf.StagePendingSubscription(PendingStreamSubscription{ + StreamID: streamID, + StartOffset: attrs.GetStartOffset(), + AlreadySubscribed: already, + Event: wf.ReserveStreamSubscribedEvent( + streamID, opts.WorkflowTaskCompletedEventID), + }) + return nil +} + +// streamSubscribedEvent is the event a subscription writes. +// +// It is recorded once per subscription, not per message: the offsets a task +// consumed ride WorkflowTaskCompleted, and payloads never enter History. The +// event exists because a command that produces none desynchronises the +// command-to-event matching every SDK's replay depends on, and because without +// it nothing in History explains why a workflow started receiving stream data. +type streamSubscribedEvent struct{} + +func (streamSubscribedEvent) Type() enumspb.EventType { + return enumspb.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED +} + +func (streamSubscribedEvent) IsWorkflowTaskTrigger() bool { return false } + +// The cursor lives in CHASM state, which is persisted and rebuilt with the +// execution, so there is nothing for replication or reset to reconstruct here. +func (streamSubscribedEvent) Apply(chasm.MutableContext, *Workflow, *historypb.HistoryEvent) error { + return nil +} + +// A command event, so it is never cherry-picked: the workflow reissues the +// subscribe command on the new branch if it still wants one. +func (streamSubscribedEvent) CherryPick( + chasm.MutableContext, + *Workflow, + *historypb.HistoryEvent, + map[enumspb.ResetReapplyExcludeType]struct{}, +) error { + return ErrEventNotCherryPickable +} + +// ReserveStreamSubscribedEvent writes the event a subscribe command owes, +// leaving the start offset for the flush to fill in. +// +// Written here rather than where the offset becomes known, because an SDK +// matches the commands a workflow issued against the events they produced by +// position. The flush runs after every command, so an event written there +// would sit behind the events of commands that were issued later, and the +// first replay of a workflow that subscribed before doing anything else would +// fail on the mismatch. +func (w *Workflow) ReserveStreamSubscribedEvent( + streamID string, + workflowTaskCompletedEventID int64, +) *historypb.HistoryEvent { + return w.AddHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED, func(e *historypb.HistoryEvent) { + e.Attributes = &historypb.HistoryEvent_WorkflowStreamSubscribedEventAttributes{ + WorkflowStreamSubscribedEventAttributes: &historypb.WorkflowStreamSubscribedEventAttributes{ + WorkflowTaskCompletedEventId: workflowTaskCompletedEventID, + StreamId: streamID, + }, + } + }) +} + +// RecordStreamSubscribedOffset completes a reserved event. Safe to do after the +// fact because the builder serializes the batch at commit, which is after the +// flush that resolves the offset. +func RecordStreamSubscribedOffset(event *historypb.HistoryEvent, startOffset int64) { + event.GetWorkflowStreamSubscribedEventAttributes().StartOffset = startOffset +} + +// streamMessagesAddedEvent is the event a publish writes. +// +// One per batch, holding the offset range and nothing else. That is what makes +// it a fixed cost: a batch of one 20-byte message and a batch of a thousand +// 2KB messages write the same event, because the bodies went to the stream's +// log. It exists for the same reason the subscription event does, that a +// command producing no event desynchronises the command-to-event matching +// every SDK's replay depends on, and it doubles as the only record in History +// that the workflow published at all. +type streamMessagesAddedEvent struct{} + +func (streamMessagesAddedEvent) Type() enumspb.EventType { + return enumspb.EVENT_TYPE_WORKFLOW_STREAM_MESSAGES_ADDED +} + +func (streamMessagesAddedEvent) IsWorkflowTaskTrigger() bool { return false } + +// The frontier it describes is CHASM state, persisted and rebuilt with the +// execution, so there is nothing here to reconstruct. +func (streamMessagesAddedEvent) Apply(chasm.MutableContext, *Workflow, *historypb.HistoryEvent) error { + return nil +} + +// A command event, so it is never cherry-picked: the offsets belong to a log +// the new branch did not write. +func (streamMessagesAddedEvent) CherryPick( + chasm.MutableContext, + *Workflow, + *historypb.HistoryEvent, + map[enumspb.ResetReapplyExcludeType]struct{}, +) error { + return ErrEventNotCherryPickable +} + +// RecordStreamMessagesAdded writes the event for one published batch. +func (w *Workflow) RecordStreamMessagesAdded( + streamID string, + firstOffset int64, + count int64, + workflowTaskCompletedEventID int64, +) { + w.AddHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_STREAM_MESSAGES_ADDED, func(e *historypb.HistoryEvent) { + e.Attributes = &historypb.HistoryEvent_WorkflowStreamMessagesAddedEventAttributes{ + WorkflowStreamMessagesAddedEventAttributes: &historypb.WorkflowStreamMessagesAddedEventAttributes{ + WorkflowTaskCompletedEventId: workflowTaskCompletedEventID, + StreamId: streamID, + FirstOffset: firstOffset, + MessageCount: count, + }, + } + }) +} + +// streamNamed returns the workflow's stream of that name, creating it on first +// use. Implicit creation is deliberate: a workflow publishing to its own output +// should not have to coordinate with anyone about who creates it. +func (w *Workflow) streamNamed(ctx chasm.MutableContext, name string) (*stream.Stream, error) { + if w.Streams == nil { + w.Streams = make(chasm.Map[string, *stream.Stream]) + } + if field, ok := w.Streams[name]; ok { + return field.Get(ctx), nil + } + + // Checked only on the create path, so an existing stream is never refused + // for room. The name arrives from the caller and every distinct one adds a + // component to this execution's mutable state, so without a bound an + // outside writer can grow that state until the size limit terminates the + // workflow. + if len(name) > stream.MaxStreamNameLength { + return nil, serviceerror.NewInvalidArgumentf( + "stream name is %d characters, over the %d limit", len(name), stream.MaxStreamNameLength) + } + if len(w.Streams) >= stream.MaxOwnedStreamsPerWorkflow { + return nil, serviceerror.NewFailedPreconditionf( + "workflow already owns %d streams, the limit", stream.MaxOwnedStreamsPerWorkflow) + } + + // Keyed on the execution so the identity is stable for the workflow, and + // distinct from any other workflow reusing the same name. + created, err := stream.NewStream(ctx, stream.NewStreamRequest{ + CollectionID: ctx.ExecutionKey().RunID + "/" + name, + Attached: true, + }) + if err != nil { + return nil, err + } + w.Streams[name] = chasm.NewComponentField(ctx, created) + return created, nil +} + +func toLibraryMessages(in []*streampb.StreamMessage) []*streamlib.StreamMessage { + out := make([]*streamlib.StreamMessage, len(in)) + for i, m := range in { + out[i] = &streamlib.StreamMessage{ + Body: m.GetBody(), + Metadata: m.GetMetadata(), + Topic: m.GetTopic(), + TopicSequence: m.GetTopicSequence(), + Kind: streamlib.STREAM_MESSAGE_KIND_DATA, + } + } + return out +} + +// streamLibrary registers the stream command with the workflow registry. +type streamLibrary struct{} + +func (l *streamLibrary) CommandHandlers() map[enumspb.CommandType]CommandHandler { + return map[enumspb.CommandType]CommandHandler{ + enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES: handleAddStreamMessagesCommand, + enumspb.COMMAND_TYPE_SUBSCRIBE_STREAM: handleSubscribeStreamCommand, + } +} + +func (l *streamLibrary) EventDefinitions() []EventDefinition { + return []EventDefinition{streamSubscribedEvent{}, streamMessagesAddedEvent{}} +} diff --git a/chasm/lib/workflow/stream_cursor_test.go b/chasm/lib/workflow/stream_cursor_test.go new file mode 100644 index 00000000000..294b2c35db2 --- /dev/null +++ b/chasm/lib/workflow/stream_cursor_test.go @@ -0,0 +1,239 @@ +package workflow + +import ( + "testing" + + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + streampb "go.temporal.io/api/stream/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + streamlib "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" +) + +func newStreamCursorTestContext() chasm.MutableContext { + return &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleExecutionKey: func() chasm.ExecutionKey { + return chasm.ExecutionKey{ + NamespaceID: "ns-1", + BusinessID: "wf-1", + RunID: "run-1", + } + }, + }, + } +} + +// Built directly rather than through NewStream, which wires a visibility field +// needing a live context. +func newAttachedStream(t *testing.T, ctx chasm.MutableContext, count int) *stream.Stream { + t.Helper() + + s := &stream.Stream{ + State: &streamlib.StreamState{ + CollectionId: "col-1", + BucketSize: stream.DefaultBucketSize, + Producers: make(map[string]*streamlib.ProducerCursor), + Consumers: make(map[string]*streamlib.ConsumerCursor), + }, + } + + messages := make([]*streamlib.StreamMessage, count) + for i := range messages { + messages[i] = &streamlib.StreamMessage{Kind: streamlib.STREAM_MESSAGE_KIND_DATA} + } + _, err := s.AddMessages(ctx, stream.AddMessagesRequest{Messages: messages}) + require.NoError(t, err) + + return s +} + +// Subscribing registers the consumer on the stream, which decides both whether +// an append is worth waking it for and what retention may not take. A range +// this workflow consumes goes into its History, and a replay is asked to +// reproduce it, so the stream holds those messages while the subscription is +// active and releases them when it is not. +func TestSubscribeRegistersTheConsumer(t *testing.T) { + ctx := newStreamCursorTestContext() + w := &Workflow{} + owned := newAttachedStream(t, ctx, 4) + w.Streams = chasm.Map[string, *stream.Stream]{ + DefaultStreamName: chasm.NewComponentField(ctx, owned), + } + + start, err := w.SubscribeToOwnedStream(ctx, DefaultStreamName, 0) + require.NoError(t, err) + require.Equal(t, int64(0), start) + + consumer := owned.State.GetConsumers()[streamConsumerID(DefaultStreamName)] + require.NotNil(t, consumer, "the stream has to know who is reading it") + require.Equal(t, int64(0), consumer.GetOffset()) + require.True(t, consumer.GetActive()) + + require.Equal(t, int64(0), consumer.GetReplayFloor()) + + // And it holds the floor for as long as it is subscribed. + require.ErrorContains(t, owned.Truncate(ctx, 1), "still depends on offset 0") +} + +func TestSubscribeFromTheTailResolvesToHead(t *testing.T) { + ctx := newStreamCursorTestContext() + w := &Workflow{} + owned := newAttachedStream(t, ctx, 4) + w.Streams = chasm.Map[string, *stream.Stream]{ + DefaultStreamName: chasm.NewComponentField(ctx, owned), + } + + start, err := w.SubscribeToOwnedStream(ctx, DefaultStreamName, -1) + require.NoError(t, err) + require.Equal(t, int64(4), start, "a negative offset means from wherever the stream is now") +} + +func TestSubscribeRejectsAStreamTheWorkflowDoesNotOwn(t *testing.T) { + ctx := newStreamCursorTestContext() + w := &Workflow{} + + _, err := w.SubscribeToOwnedStream(ctx, "absent", 0) + require.ErrorContains(t, err, "does not own a stream") +} + +// Committing a delivered range moves the consumer's cursor on the stream, so +// the stream knows how far this reader has got. +func TestCommitStreamCursorsAdvancesTheConsumer(t *testing.T) { + ctx := newStreamCursorTestContext() + w := &Workflow{} + owned := newAttachedStream(t, ctx, 4) + w.Streams = chasm.Map[string, *stream.Stream]{ + DefaultStreamName: chasm.NewComponentField(ctx, owned), + } + + _, err := w.SubscribeToOwnedStream(ctx, DefaultStreamName, 0) + require.NoError(t, err) + + cursor := w.StreamCursors[DefaultStreamName].Get(ctx) + require.NoError(t, cursor.StagePending(ctx, 0, 3)) + + recorded := w.CommitStreamCursors(ctx) + require.Len(t, recorded, 1) + require.Equal(t, int64(0), recorded[0].GetFromOffset()) + require.Equal(t, int64(3), recorded[0].GetToOffset()) + + require.Equal(t, int64(3), + owned.State.GetConsumers()[streamConsumerID(DefaultStreamName)].GetOffset(), + "the stream must see how far the consumer has read") +} + +// An idle task records an empty range, which must leave the consumer alone. +func TestCommitStreamCursorsWithAnEmptyRangeHoldsTheConsumer(t *testing.T) { + ctx := newStreamCursorTestContext() + w := &Workflow{} + owned := newAttachedStream(t, ctx, 4) + w.Streams = chasm.Map[string, *stream.Stream]{ + DefaultStreamName: chasm.NewComponentField(ctx, owned), + } + + _, err := w.SubscribeToOwnedStream(ctx, DefaultStreamName, 0) + require.NoError(t, err) + + cursor := w.StreamCursors[DefaultStreamName].Get(ctx) + require.NoError(t, cursor.StagePending(ctx, 0, 0)) + + recorded := w.CommitStreamCursors(ctx) + require.Len(t, recorded, 1, "an empty range is still recorded") + require.Equal(t, recorded[0].GetFromOffset(), recorded[0].GetToOffset()) + + require.Equal(t, int64(0), + owned.State.GetConsumers()[streamConsumerID(DefaultStreamName)].GetOffset(), + "consuming nothing must not move the consumer") +} + +// Two publishes in one workflow task each stage their own batch, at the offsets +// they landed at. +// +// The offset a batch starts at is the key its row is written under, so two +// publishes must not collide and a retry of either must address the same row +// it wrote before. There is no transaction id involved any more: the store +// resolves a rewrite by replacing, and the frontier decides what a reader sees. +func TestPublishStagesEachBatchAtItsOwnOffset(t *testing.T) { + ctx := newStreamCursorTestContext() + + // A backend, because the handler writes the publish event and that event is + // part of what the command owes. + backend := &chasm.MockNodeBackend{ + HandleAddHistoryEvent: func( + t enumspb.EventType, set func(*historypb.HistoryEvent), + ) *historypb.HistoryEvent { + e := &historypb.HistoryEvent{EventType: t} + set(e) + return e + }, + } + w := &Workflow{MSPointer: chasm.NewMSPointer(backend)} + opts := CommandHandlerOptions{WorkflowTaskCompletedEventID: 10} + + publish := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("x")}}, + {Body: &commonpb.Payload{Data: []byte("y")}}, + }, + }, + }, + } + + require.NoError(t, handleAddStreamMessagesCommand(ctx, w, allowAnySize{}, publish, opts)) + require.NoError(t, handleAddStreamMessagesCommand(ctx, w, allowAnySize{}, publish, opts)) + + // Both publishes committed with the workflow task, so the batches are on + // the component keyed by the offsets they start at. + owned := w.Streams[DefaultStreamName].Get(ctx) + require.Len(t, owned.Batches, 2) + _, ok := owned.Batches[0] + require.True(t, ok) + _, ok = owned.Batches[2] + require.True(t, ok) + require.Equal(t, int64(4), owned.State.GetHeadOffset()) +} + +type allowAnySize struct{} + +func (allowAnySize) IsValidPayloadSize(int) bool { return true } + +// A consumer that falls behind a truncating stream must be told, not handed +// what is left with a hole in it. +// +// Being outrun is possible again once the subscription is released, which is +// deliberate: a floor that nothing ever gave up would keep every message for +// as long as a consumer had ever existed. The trade is that being outrun has +// to be loud. +func TestConsumerOutrunByTruncationIsToldSo(t *testing.T) { + ctx := newStreamCursorTestContext() + w := &Workflow{} + owned := newAttachedStream(t, ctx, 4) + w.Streams = chasm.Map[string, *stream.Stream]{ + DefaultStreamName: chasm.NewComponentField(ctx, owned), + } + + _, err := w.SubscribeToOwnedStream(ctx, DefaultStreamName, 0) + require.NoError(t, err) + + // Someone decides these messages are no longer needed, and only then can + // the stream move past where this consumer is sitting. + owned.DeregisterConsumer(ctx, streamConsumerID(DefaultStreamName)) + err = owned.Truncate(ctx, 3) + require.NoError(t, err, "a released floor must not keep holding") + + cursor := w.StreamCursors[DefaultStreamName].Get(ctx) + require.Less(t, cursor.Offset(), owned.State.GetBaseOffset()) + + state, err := owned.Snapshot(ctx, struct{}{}) + require.NoError(t, err) + require.Equal(t, int64(3), state.GetBaseOffset(), + "the floor moved, which is what the consumer has to find out about") +} diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index 2b5409a4147..053bc452086 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -2,15 +2,20 @@ package workflow import ( "fmt" + "slices" + "strings" commonpb "go.temporal.io/api/common/v1" failurepb "go.temporal.io/api/failure/v1" historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" + streampb "go.temporal.io/api/stream/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/callback" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/chasm/lib/nexusoperation" + "go.temporal.io/server/chasm/lib/stream" + streamlib "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" chasmworkflowpb "go.temporal.io/server/chasm/lib/workflow/gen/workflowpb/v1" "go.temporal.io/server/service/history/historybuilder" "google.golang.org/protobuf/types/known/emptypb" @@ -39,6 +44,294 @@ type Workflow struct { // Updates indexed by update ID, used to store the update components. Updates chasm.Map[string, *WorkflowUpdate] + + // Streams the workflow owns, keyed by stream name. Co-located with the + // workflow so publishing rides its commit rather than crossing executions. + Streams chasm.Map[string, *stream.Stream] + + // Positions in streams the workflow consumes, keyed by stream name. Held + // here rather than on the stream so that folding in a delivered range + // commits with the event that records it. + StreamCursors chasm.Map[string, *stream.Cursor] + + // Subscribe commands whose stream is in another execution, so the addressing + // has to be looked up before a cursor can be made. In memory only, drained + // by the flush before commit. + pendingStreamSubscriptions []PendingStreamSubscription +} + +// PendingStreamSubscription is a subscribe command whose stream lives in +// another execution, waiting for the flush to look up its addressing. +type PendingStreamSubscription struct { + StreamID string + StartOffset int64 + // The workflow already holds a cursor for this stream. The subscription + // itself is done, but the command still needs its event, because that is + // what a replaying worker matches the re-issued command against. + AlreadySubscribed bool + + // The event this command already wrote, waiting on its start offset. In + // memory only, like the rest of this struct. + Event *historypb.HistoryEvent +} + +// StagePendingSubscription records a subscription for the flush to resolve. +func (w *Workflow) StagePendingSubscription(sub PendingStreamSubscription) { + w.pendingStreamSubscriptions = append(w.pendingStreamSubscriptions, sub) +} + +// DrainStreamSubscriptions returns and clears the staged subscriptions. +func (w *Workflow) DrainStreamSubscriptions() []PendingStreamSubscription { + out := w.pendingStreamSubscriptions + w.pendingStreamSubscriptions = nil + return out +} + +// streamConsumerID names this workflow's pin on a stream it owns. An attached +// stream has exactly one consumer, but the stream's map is keyed by consumer, +// so the entry still needs a stable name. +func streamConsumerID(streamName string) string { + return "workflow:" + streamName +} + +// SubscribeToOwnedStream registers this workflow as a consumer of a stream it +// owns, returning the offset the subscription actually starts from. +// +// A negative start offset means "from wherever the stream is now". That is +// resolved here and stored, so the first recorded range begins at a fact rather +// than at a reading that would land somewhere else on replay. +func (w *Workflow) SubscribeToOwnedStream( + mctx chasm.MutableContext, + name string, + startOffset int64, +) (int64, error) { + field, ok := w.Streams[name] + if !ok { + return 0, serviceerror.NewNotFoundf("workflow does not own a stream named %q", name) + } + owned := field.Get(mctx) + state, err := owned.Snapshot(mctx, struct{}{}) + if err != nil { + return 0, err + } + + if startOffset < 0 { + startOffset = state.GetHeadOffset() + } + if startOffset < state.GetBaseOffset() { + return 0, serviceerror.NewFailedPreconditionf( + "offset %d is below the stream's floor of %d", startOffset, state.GetBaseOffset()) + } + + if w.StreamCursors == nil { + w.StreamCursors = make(chasm.Map[string, *stream.Cursor]) + } + if existing, ok := w.StreamCursors[name]; ok { + // Resubscribing must not rewind a cursor: ranges below it are already + // recorded in History, and moving back would replay them as new. + return existing.Get(mctx).Offset(), nil + } + + cursor, err := stream.NewCursor(mctx, stream.NewCursorRequest{ + StreamID: name, + CollectionID: state.GetCollectionId(), + BucketSize: state.GetBucketSize(), + StartOffset: startOffset, + }) + if err != nil { + return 0, err + } + + // Pin the stream's floor in the same transaction. Registered separately it + // could be lost while the cursor survived, and truncation would then be + // free to take a range the cursor still points at. + key := mctx.ExecutionKey() + if err := owned.RegisterConsumer(mctx, streamConsumerID(name), key.BusinessID, key.RunID, startOffset, false); err != nil { + return 0, err + } + + w.StreamCursors[name] = chasm.NewComponentField(mctx, cursor) + return startOffset, nil +} + +// ExternalStreamSubscription describes a stream in another execution. The +// addressing is copied in at subscribe time so delivery never has to reach +// across executions to find the log. +type ExternalStreamSubscription struct { + StreamID string + CollectionID string + BucketSize int64 + StartOffset int64 + KnownHead int64 +} + +// SubscribeToExternalStream registers this workflow as a consumer of a stream +// it does not own, returning the offset the subscription starts from. +func (w *Workflow) SubscribeToExternalStream( + mctx chasm.MutableContext, + req ExternalStreamSubscription, +) (int64, error) { + if w.StreamCursors == nil { + w.StreamCursors = make(chasm.Map[string, *stream.Cursor]) + } + if existing, ok := w.StreamCursors[req.StreamID]; ok { + // Resubscribing must not rewind: ranges below the cursor are already + // recorded in History, and moving back would replay them as new. + return existing.Get(mctx).Offset(), nil + } + + cursor, err := stream.NewCursor(mctx, stream.NewCursorRequest{ + StreamID: req.StreamID, + External: true, + CollectionID: req.CollectionID, + BucketSize: req.BucketSize, + StartOffset: req.StartOffset, + }) + if err != nil { + return 0, err + } + cursor.AdvanceKnownHead(mctx, req.KnownHead) + w.StreamCursors[req.StreamID] = chasm.NewComponentField(mctx, cursor) + return req.StartOffset, nil +} + +// ExportStreamSubscriptions returns the subscriptions a successor run has to +// inherit. +// +// Only subscriptions to streams in other executions are exported. A stream this +// workflow owns lives in this execution and does not itself survive the run +// transition yet (§8a), so carrying a cursor for one would leave the successor +// pointing at a stream it cannot reach. +func (w *Workflow) ExportStreamSubscriptions(ctx chasm.Context) []ExternalStreamSubscription { + var out []ExternalStreamSubscription + for _, field := range w.StreamCursors { + cursor := field.Get(ctx) + if !cursor.IsExternal() { + continue + } + out = append(out, ExternalStreamSubscription{ + StreamID: cursor.StreamID(), + CollectionID: cursor.CollectionID(), + BucketSize: cursor.BucketSize(), + StartOffset: cursor.Offset(), + KnownHead: cursor.KnownHead(), + }) + } + // Stable order, so a successor's state does not depend on map iteration. + slices.SortFunc(out, func(a, b ExternalStreamSubscription) int { + return strings.Compare(a.StreamID, b.StreamID) + }) + return out +} + +// ImportStreamSubscriptions installs subscriptions inherited from the run this +// one continues. The offset carries over unchanged: offsets are global to the +// stream, so the successor resumes exactly where its predecessor stopped and +// the stream itself is untouched. +func (w *Workflow) ImportStreamSubscriptions( + mctx chasm.MutableContext, + subscriptions []ExternalStreamSubscription, +) error { + for _, sub := range subscriptions { + if _, err := w.SubscribeToExternalStream(mctx, sub); err != nil { + return err + } + } + return nil +} + +// AdvanceKnownHead records how far a stream in another execution has moved. +// +// A workflow cannot read that frontier itself while closing its own +// transaction, so the stream pushes it here. Writing it dirties this execution, +// and the transaction close then sees the cursor is behind and schedules a +// workflow task, which is the same path an owned stream takes. +func (w *Workflow) AdvanceKnownHead(mctx chasm.MutableContext, streamID string, head int64) error { + field, ok := w.StreamCursors[streamID] + if !ok { + return serviceerror.NewNotFoundf("workflow does not consume stream %q", streamID) + } + field.Get(mctx).AdvanceKnownHead(mctx, head) + return nil +} + +// StreamCursorsBehind reports whether any subscription still has offsets it has +// not been given. +// +// This is the one place a stream wakes a workflow. Publishing deliberately +// never does, because a stream item is data produced by an execution rather +// than a decision input to it. An active subscription is different: the +// workflow asked to be told, so leaving it to wait for some unrelated task +// would make delivery depend on traffic that has nothing to do with the stream. +func (w *Workflow) StreamCursorsBehind(ctx chasm.Context) bool { + for name, field := range w.StreamCursors { + cursor := field.Get(ctx) + + // An external stream's frontier is whatever it last pushed here: this + // execution cannot read the real one without reaching into another. + if cursor.IsExternal() { + if cursor.Offset() < cursor.KnownHead() { + return true + } + continue + } + + owned, ok := w.Streams[name] + if !ok { + continue + } + state, err := owned.Get(ctx).Snapshot(ctx, struct{}{}) + if err != nil { + continue + } + if cursor.Offset() < state.GetHeadOffset() { + return true + } + } + return false +} + +// CommitStreamCursors folds every staged range into its cursor and returns the +// ranges to record. Called while the workflow task's transaction is open, so +// the advance and the event that carries the range land together. +// +// A cursor with nothing staged is skipped, but a cursor staged with an empty +// range is not: replay has to see that the subscription was live and observed +// nothing. +func (w *Workflow) CommitStreamCursors(mctx chasm.MutableContext) []*streampb.StreamCursor { + if w.StreamCursors == nil { + return nil + } + + names := make([]string, 0, len(w.StreamCursors)) + for name := range w.StreamCursors { + names = append(names, name) + } + // Recorded order has to be stable, or replay compares against a different + // event than the one the original execution wrote. + slices.Sort(names) + + var recorded []*streampb.StreamCursor + for _, name := range names { + cursor := w.StreamCursors[name].Get(mctx) + from, to, ok := cursor.Commit(mctx) + if !ok { + continue + } + + // Let the floor follow the cursor. Anything below it is recorded as + // consumed, so nothing needs to re-read it. + if field, ok := w.Streams[name]; ok { + field.Get(mctx).AdvanceConsumer(mctx, streamConsumerID(name), to) + } + + recorded = append(recorded, &streampb.StreamCursor{ + StreamId: cursor.StreamID(), + FromOffset: from, + ToOffset: to, + }) + } + return recorded } func NewWorkflow( @@ -317,3 +610,76 @@ func (w *Workflow) HasAnyBufferedEvent(filter historybuilder.BufferedEventFilter func (w *Workflow) WorkflowTypeName() string { return w.GetWorkflowTypeName() } + +// OwnedStreamState returns the state of a stream this workflow owns, or nil if +// it owns none by that name. +// +// An attached stream has no id of its own, so this is the only way to see its +// frontier from outside the execution. Reading it needs the owner's component, +// which is why it lives here rather than on the stream. +// +// Absent is not an error. An owned stream is created by the first publish to +// it, so a reader that arrives before the workflow has published anything is +// the ordinary case rather than a mistake, and from outside the execution +// "not created yet" and "never will be" are the same observation. +func (w *Workflow) OwnedStreamState( + ctx chasm.Context, + name string, +) (*streamlib.StreamState, error) { + field, ok := w.Streams[name] + if !ok { + return nil, nil + } + return field.Get(ctx).Snapshot(ctx, struct{}{}) +} + +// OwnedStream returns the attached stream itself, or nil when the workflow has +// not created it yet. Reads that need the payload and not just the frontier go +// through here, so both come from one view of the component. +func (w *Workflow) OwnedStream( + ctx chasm.Context, + name string, +) *stream.Stream { + field, ok := w.Streams[name] + if !ok { + return nil + } + return field.Get(ctx) +} + +// EnsureOwnedStream creates a stream this workflow owns if the first writer to +// it is not the workflow itself, and returns its state either way. +// +// An outside writer needs the stream's collection id and bucket size to write +// its log node, and both are decided when the stream is created. So the first +// append from outside costs one transition to create the stream and learn +// them, and none after that. +func (w *Workflow) EnsureOwnedStream( + mctx chasm.MutableContext, + name string, +) (*streamlib.StreamState, error) { + s, err := w.streamNamed(mctx, name) + if err != nil { + return nil, err + } + return s.Snapshot(mctx, struct{}{}) +} + +// AppendToOwnedStream appends to a stream this workflow owns on behalf of a +// writer outside the execution. +// +// The workflow's own publishes go through the command handler, which advances +// the frontier inside the Workflow Task's commit. This is the other producer: +// it advances the same frontier in a transition of its own, so the two are +// serialized by the execution rather than by anything the stream does. +func (w *Workflow) AppendToOwnedStream( + mctx chasm.MutableContext, + name string, + req stream.AddMessagesRequest, +) (stream.AddMessagesResult, error) { + s, err := w.streamNamed(mctx, name) + if err != nil { + return stream.AddMessagesResult{}, err + } + return s.AddMessages(mctx, req) +} diff --git a/cmd/tools/getproto/files.go b/cmd/tools/getproto/files.go index 38b6ef67622..04b99e22706 100644 --- a/cmd/tools/getproto/files.go +++ b/cmd/tools/getproto/files.go @@ -27,6 +27,7 @@ import ( rules "go.temporal.io/api/rules/v1" schedule "go.temporal.io/api/schedule/v1" sdk "go.temporal.io/api/sdk/v1" + stream "go.temporal.io/api/stream/v1" taskqueue "go.temporal.io/api/taskqueue/v1" update "go.temporal.io/api/update/v1" version "go.temporal.io/api/version/v1" @@ -91,6 +92,7 @@ func init() { importMap["temporal/api/sdk/v1/task_complete_metadata.proto"] = sdk.File_temporal_api_sdk_v1_task_complete_metadata_proto importMap["temporal/api/sdk/v1/user_metadata.proto"] = sdk.File_temporal_api_sdk_v1_user_metadata_proto importMap["temporal/api/sdk/v1/worker_config.proto"] = sdk.File_temporal_api_sdk_v1_worker_config_proto + importMap["temporal/api/stream/v1/message.proto"] = stream.File_temporal_api_stream_v1_message_proto importMap["temporal/api/taskqueue/v1/message.proto"] = taskqueue.File_temporal_api_taskqueue_v1_message_proto importMap["temporal/api/update/v1/message.proto"] = update.File_temporal_api_update_v1_message_proto importMap["temporal/api/version/v1/message.proto"] = version.File_temporal_api_version_v1_message_proto diff --git a/cmd/tools/protogen/main.go b/cmd/tools/protogen/main.go index cf2c00b2e6e..251f2a26adb 100644 --- a/cmd/tools/protogen/main.go +++ b/cmd/tools/protogen/main.go @@ -165,16 +165,65 @@ func newGenerator() (*generator, error) { return &gen, nil } +// removeExistingGenDirs clears out the previous run's output so a proto that +// was deleted does not leave its Go behind. +// +// Only generated files go. A gen package sometimes needs a hand-written file +// beside the generated ones, because a method has to be declared in the same +// package as the type it is on, and wiping the directory deleted those without +// saying so. Empty directories are then pruned, which is what removing a proto +// used to rely on. func (g *generator) removeExistingGenDirs() error { for _, dir := range g.chasmLibDirs { genDir := filepath.Join(dir, "gen") - if err := os.RemoveAll(genDir); err != nil { - return fmt.Errorf("error removing directory %s: %w", genDir, err) + if !exists(genDir) { + continue + } + if err := filepath.Walk(genDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !strings.HasSuffix(info.Name(), ".pb.go") { + return nil + } + return os.Remove(path) + }); err != nil { + return fmt.Errorf("error removing generated files under %s: %w", genDir, err) + } + if err := pruneEmptyDirs(genDir); err != nil { + return fmt.Errorf("error pruning empty directories under %s: %w", genDir, err) } } return nil } +// pruneEmptyDirs removes dir and any descendant left with nothing in it, +// deepest first. +func pruneEmptyDirs(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, entry := range entries { + if entry.IsDir() { + if err := pruneEmptyDirs(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + } + remaining, err := os.ReadDir(dir) + if err != nil { + return err + } + if len(remaining) == 0 { + return os.Remove(dir) + } + return nil +} + func (g *generator) backupProtos() error { if exists(g.protoOut) { if err := os.Rename(g.protoOut, g.protoBackup); err != nil { diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 3b845ef2358..835400d996f 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -98,6 +98,12 @@ const ( PersistenceAppendHistoryNodesScope = "AppendHistoryNodes" // PersistenceAppendRawHistoryNodesScope tracks AppendRawHistoryNodes calls made by service to persistence layer PersistenceAppendRawHistoryNodesScope = "AppendRawHistoryNodes" + // PersistenceAppendStreamLogScope tracks AppendStreamLog calls made by service to persistence layer + PersistenceAppendStreamLogScope = "AppendStreamLog" + // PersistenceReadStreamLogScope tracks ReadStreamLog calls made by service to persistence layer + PersistenceReadStreamLogScope = "ReadStreamLog" + // PersistenceDeleteStreamLogBucketScope tracks DeleteStreamLogBucket calls made by service to persistence layer + PersistenceDeleteStreamLogBucketScope = "DeleteStreamLogBucket" // PersistenceReadHistoryBranchScope tracks ReadHistoryBranch calls made by service to persistence layer PersistenceReadHistoryBranchScope = "ReadHistoryBranch" // PersistenceReadHistoryBranchReverseScope tracks ReadHistoryBranchReverse calls made by service to persistence layer diff --git a/common/persistence/cassandra/history_store.go b/common/persistence/cassandra/history_store.go index a2069e5a5f4..9a5a5c3a538 100644 --- a/common/persistence/cassandra/history_store.go +++ b/common/persistence/cassandra/history_store.go @@ -429,3 +429,102 @@ func convertTimeoutError(err error) error { } return err } + +const ( + // Upsert by nature in Cassandra, which is exactly the semantics wanted: the + // key is the offset a batch starts at, so a retry replaces rather than + // competes. There is no transaction-id chain to order writers against. + templateUpsertStreamLog = `INSERT INTO stream_log (` + + `shard_id, namespace_id, collection_id, bucket, start_offset, next_offset, data, data_encoding) ` + + `VALUES (?, ?, ?, ?, ?, ?, ?, ?) ` + + // Two reads rather than a subquery, which Cassandra has no notion of. The + // first is a reverse slice of one row to find the batch containing the + // requested offset; the second reads forward from there. + templateSelectStreamLogFloor = `SELECT start_offset FROM stream_log ` + + `WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ? ` + + `AND start_offset <= ? ORDER BY start_offset DESC LIMIT 1 ` + + templateSelectStreamLog = `SELECT start_offset, data, data_encoding FROM stream_log ` + + `WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ? ` + + `AND start_offset >= ? AND start_offset < ? ` + + templateDeleteStreamLogBucket = `DELETE FROM stream_log ` + + `WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ? ` +) + +// AppendStreamLog writes one batch of a stream log. +func (h *HistoryStore) AppendStreamLog( + ctx context.Context, + request *p.InternalAppendStreamLogRequest, +) error { + query := h.Session.Query(templateUpsertStreamLog, + request.ShardID, + request.NamespaceID, + request.CollectionID, + request.Bucket, + request.StartOffset, + request.NextOffset, + request.Node.Data, + request.Node.EncodingType.String(), + ).WithContext(ctx) + if err := query.Exec(); err != nil { + return gocql.ConvertError("AppendStreamLog", err) + } + return nil +} + +// ReadStreamLog returns the batches covering the requested range, beginning +// with the batch that contains MinOffset. +func (h *HistoryStore) ReadStreamLog( + ctx context.Context, + request *p.InternalReadStreamLogRequest, +) (*p.InternalReadStreamLogResponse, error) { + from := request.MinOffset + var floor int64 + err := h.Session.Query(templateSelectStreamLogFloor, + request.ShardID, request.NamespaceID, request.CollectionID, request.Bucket, request.MinOffset, + ).WithContext(ctx).Scan(&floor) + switch { + case err == nil: + from = floor + case gocql.IsNotFoundError(err): + // Nothing at or below, so the range starts wherever it starts. + default: + return nil, gocql.ConvertError("ReadStreamLog", err) + } + + iter := h.Session.Query(templateSelectStreamLog, + request.ShardID, request.NamespaceID, request.CollectionID, request.Bucket, + from, request.MaxOffset, + ).WithContext(ctx).PageSize(request.PageSize).Iter() + + resp := &p.InternalReadStreamLogResponse{} + var startOffset int64 + var data []byte + var encoding string + for iter.Scan(&startOffset, &data, &encoding) { + resp.Batches = append(resp.Batches, p.NewDataBlob(data, encoding)) + resp.StartOffsets = append(resp.StartOffsets, startOffset) + data = nil + } + if err := iter.Close(); err != nil { + return nil, gocql.ConvertError("ReadStreamLog", err) + } + return resp, nil +} + +// DeleteStreamLogBucket drops a whole bucket, which on Cassandra is one +// partition and so one tombstone rather than a row-by-row delete. +func (h *HistoryStore) DeleteStreamLogBucket( + ctx context.Context, + request *p.InternalDeleteStreamLogBucketRequest, +) error { + query := h.Session.Query(templateDeleteStreamLogBucket, + request.ShardID, request.NamespaceID, request.CollectionID, request.Bucket, + ).WithContext(ctx) + if err := query.Exec(); err != nil { + return gocql.ConvertError("DeleteStreamLogBucket", err) + } + return nil +} diff --git a/common/persistence/data_interfaces.go b/common/persistence/data_interfaces.go index c818b447866..ab15e76cdb9 100644 --- a/common/persistence/data_interfaces.go +++ b/common/persistence/data_interfaces.go @@ -1162,6 +1162,13 @@ type ( AppendHistoryNodes(ctx context.Context, request *AppendHistoryNodesRequest) (*AppendHistoryNodesResponse, error) // AppendRawHistoryNodes add a node of raw histories to history node table AppendRawHistoryNodes(ctx context.Context, request *AppendRawHistoryNodesRequest) (*AppendHistoryNodesResponse, error) + + // Stream logs. Not history: an offset-addressed sequence a stream + // component owns, where a write is idempotent by the offset its batch + // starts at. + AppendStreamLog(ctx context.Context, request *InternalAppendStreamLogRequest) error + ReadStreamLog(ctx context.Context, request *InternalReadStreamLogRequest) (*InternalReadStreamLogResponse, error) + DeleteStreamLogBucket(ctx context.Context, request *InternalDeleteStreamLogBucketRequest) error // ReadHistoryBranch returns history node data for a branch ReadHistoryBranch(ctx context.Context, request *ReadHistoryBranchRequest) (*ReadHistoryBranchResponse, error) // ReadHistoryBranchByBatch returns history node data for a branch ByBatch @@ -1433,6 +1440,23 @@ func BuildHistoryGarbageCleanupInfo(namespaceID, workflowID, runID string) strin return fmt.Sprintf("%v:%v:%v", namespaceID, workflowID, runID) } +// NonExecutionGarbageCleanupInfoPrefix marks a history branch whose rows are +// not a workflow execution's history. +// +// The scavenger finds the execution named by a branch's cleanup info and +// deletes the branch when there is none, which is right for a workflow whose +// execution is gone and wrong for a branch that never had one. Anything +// storing rows in this table for its own purposes has to say so, or the +// scavenger reclaims live data. +const NonExecutionGarbageCleanupInfoPrefix = "non-execution:" + +// IsNonExecutionGarbageCleanupInfo reports whether a branch belongs to +// something other than a workflow execution, and so is not the scavenger's to +// collect. Whatever wrote it owns deleting it. +func IsNonExecutionGarbageCleanupInfo(info string) bool { + return strings.HasPrefix(info, NonExecutionGarbageCleanupInfoPrefix) +} + // SplitHistoryGarbageCleanupInfo returns workflow identity information func SplitHistoryGarbageCleanupInfo(info string) (namespaceID, workflowID, runID string, err error) { // Expect format: namespaceID:workflowID:runID, but workflowID may contain ':' so we diff --git a/common/persistence/data_interfaces_mock.go b/common/persistence/data_interfaces_mock.go index 9ba28072499..4f19aece8cf 100644 --- a/common/persistence/data_interfaces_mock.go +++ b/common/persistence/data_interfaces_mock.go @@ -214,6 +214,20 @@ func (mr *MockExecutionManagerMockRecorder) AppendRawHistoryNodes(ctx, request a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendRawHistoryNodes", reflect.TypeOf((*MockExecutionManager)(nil).AppendRawHistoryNodes), ctx, request) } +// AppendStreamLog mocks base method. +func (m *MockExecutionManager) AppendStreamLog(ctx context.Context, request *InternalAppendStreamLogRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendStreamLog", ctx, request) + ret0, _ := ret[0].(error) + return ret0 +} + +// AppendStreamLog indicates an expected call of AppendStreamLog. +func (mr *MockExecutionManagerMockRecorder) AppendStreamLog(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendStreamLog", reflect.TypeOf((*MockExecutionManager)(nil).AppendStreamLog), ctx, request) +} + // Close mocks base method. func (m *MockExecutionManager) Close() { m.ctrl.T.Helper() @@ -312,6 +326,20 @@ func (mr *MockExecutionManagerMockRecorder) DeleteReplicationTaskFromDLQ(ctx, re return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteReplicationTaskFromDLQ", reflect.TypeOf((*MockExecutionManager)(nil).DeleteReplicationTaskFromDLQ), ctx, request) } +// DeleteStreamLogBucket mocks base method. +func (m *MockExecutionManager) DeleteStreamLogBucket(ctx context.Context, request *InternalDeleteStreamLogBucketRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteStreamLogBucket", ctx, request) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteStreamLogBucket indicates an expected call of DeleteStreamLogBucket. +func (mr *MockExecutionManagerMockRecorder) DeleteStreamLogBucket(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStreamLogBucket", reflect.TypeOf((*MockExecutionManager)(nil).DeleteStreamLogBucket), ctx, request) +} + // DeleteWorkflowExecution mocks base method. func (m *MockExecutionManager) DeleteWorkflowExecution(ctx context.Context, request *DeleteWorkflowExecutionRequest) error { m.ctrl.T.Helper() @@ -576,6 +604,21 @@ func (mr *MockExecutionManagerMockRecorder) ReadRawHistoryBranch(ctx, request an return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadRawHistoryBranch", reflect.TypeOf((*MockExecutionManager)(nil).ReadRawHistoryBranch), ctx, request) } +// ReadStreamLog mocks base method. +func (m *MockExecutionManager) ReadStreamLog(ctx context.Context, request *InternalReadStreamLogRequest) (*InternalReadStreamLogResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadStreamLog", ctx, request) + ret0, _ := ret[0].(*InternalReadStreamLogResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadStreamLog indicates an expected call of ReadStreamLog. +func (mr *MockExecutionManagerMockRecorder) ReadStreamLog(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadStreamLog", reflect.TypeOf((*MockExecutionManager)(nil).ReadStreamLog), ctx, request) +} + // SetWorkflowExecution mocks base method. func (m *MockExecutionManager) SetWorkflowExecution(ctx context.Context, request *SetWorkflowExecutionRequest) (*SetWorkflowExecutionResponse, error) { m.ctrl.T.Helper() diff --git a/common/persistence/history_manager.go b/common/persistence/history_manager.go index 9777739be73..33b46f67fd0 100644 --- a/common/persistence/history_manager.go +++ b/common/persistence/history_manager.go @@ -1138,3 +1138,29 @@ func (m *executionManagerImpl) serializeToken( return m.pagingTokenSerializer.Serialize(pagingToken) } + +// AppendStreamLog writes one batch of a stream log. A pass-through: there is no +// branch token to build, no chain to maintain and nothing to serialize, because +// the offset is the key and the blob is already encoded. +func (m *executionManagerImpl) AppendStreamLog( + ctx context.Context, + request *InternalAppendStreamLogRequest, +) error { + return m.persistence.AppendStreamLog(ctx, request) +} + +// ReadStreamLog returns the batches covering a range. +func (m *executionManagerImpl) ReadStreamLog( + ctx context.Context, + request *InternalReadStreamLogRequest, +) (*InternalReadStreamLogResponse, error) { + return m.persistence.ReadStreamLog(ctx, request) +} + +// DeleteStreamLogBucket drops a whole bucket. +func (m *executionManagerImpl) DeleteStreamLogBucket( + ctx context.Context, + request *InternalDeleteStreamLogBucketRequest, +) error { + return m.persistence.DeleteStreamLogBucket(ctx, request) +} diff --git a/common/persistence/mock/store_mock.go b/common/persistence/mock/store_mock.go index e78040bdaf3..397915450d6 100644 --- a/common/persistence/mock/store_mock.go +++ b/common/persistence/mock/store_mock.go @@ -908,6 +908,20 @@ func (mr *MockExecutionStoreMockRecorder) AppendHistoryNodes(ctx, request any) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendHistoryNodes", reflect.TypeOf((*MockExecutionStore)(nil).AppendHistoryNodes), ctx, request) } +// AppendStreamLog mocks base method. +func (m *MockExecutionStore) AppendStreamLog(ctx context.Context, request *persistence.InternalAppendStreamLogRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendStreamLog", ctx, request) + ret0, _ := ret[0].(error) + return ret0 +} + +// AppendStreamLog indicates an expected call of AppendStreamLog. +func (mr *MockExecutionStoreMockRecorder) AppendStreamLog(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendStreamLog", reflect.TypeOf((*MockExecutionStore)(nil).AppendStreamLog), ctx, request) +} + // Close mocks base method. func (m *MockExecutionStore) Close() { m.ctrl.T.Helper() @@ -1019,6 +1033,20 @@ func (mr *MockExecutionStoreMockRecorder) DeleteReplicationTaskFromDLQ(ctx, requ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteReplicationTaskFromDLQ", reflect.TypeOf((*MockExecutionStore)(nil).DeleteReplicationTaskFromDLQ), ctx, request) } +// DeleteStreamLogBucket mocks base method. +func (m *MockExecutionStore) DeleteStreamLogBucket(ctx context.Context, request *persistence.InternalDeleteStreamLogBucketRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteStreamLogBucket", ctx, request) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteStreamLogBucket indicates an expected call of DeleteStreamLogBucket. +func (mr *MockExecutionStoreMockRecorder) DeleteStreamLogBucket(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStreamLogBucket", reflect.TypeOf((*MockExecutionStore)(nil).DeleteStreamLogBucket), ctx, request) +} + // DeleteWorkflowExecution mocks base method. func (m *MockExecutionStore) DeleteWorkflowExecution(ctx context.Context, request *persistence.DeleteWorkflowExecutionRequest) error { m.ctrl.T.Helper() @@ -1252,6 +1280,21 @@ func (mr *MockExecutionStoreMockRecorder) ReadHistoryBranch(ctx, request any) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadHistoryBranch", reflect.TypeOf((*MockExecutionStore)(nil).ReadHistoryBranch), ctx, request) } +// ReadStreamLog mocks base method. +func (m *MockExecutionStore) ReadStreamLog(ctx context.Context, request *persistence.InternalReadStreamLogRequest) (*persistence.InternalReadStreamLogResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadStreamLog", ctx, request) + ret0, _ := ret[0].(*persistence.InternalReadStreamLogResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadStreamLog indicates an expected call of ReadStreamLog. +func (mr *MockExecutionStoreMockRecorder) ReadStreamLog(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadStreamLog", reflect.TypeOf((*MockExecutionStore)(nil).ReadStreamLog), ctx, request) +} + // SetWorkflowExecution mocks base method. func (m *MockExecutionStore) SetWorkflowExecution(ctx context.Context, request *persistence.InternalSetWorkflowExecutionRequest) error { m.ctrl.T.Helper() diff --git a/common/persistence/persistence_interface.go b/common/persistence/persistence_interface.go index b95801ab8df..9793205c75c 100644 --- a/common/persistence/persistence_interface.go +++ b/common/persistence/persistence_interface.go @@ -160,6 +160,20 @@ type ( DeleteHistoryBranch(ctx context.Context, request *InternalDeleteHistoryBranchRequest) error // GetHistoryTreeContainingBranch returns all branch information of the tree containing the specified branch GetHistoryTreeContainingBranch(ctx context.Context, request *InternalGetHistoryTreeContainingBranchRequest) (*InternalGetHistoryTreeContainingBranchResponse, error) + // The below are the stream log APIs. A stream log is not history: it is + // an offset-addressed sequence a stream component owns, and a write is + // idempotent by the offset its batch starts at rather than ordered + // against other writers by a transaction id. + + // AppendStreamLog writes one batch, replacing any batch already at that + // offset. A retry of an append addresses the same row. + AppendStreamLog(ctx context.Context, request *InternalAppendStreamLogRequest) error + // ReadStreamLog returns the batches covering a range, beginning with the + // batch that contains the first offset asked for. + ReadStreamLog(ctx context.Context, request *InternalReadStreamLogRequest) (*InternalReadStreamLogResponse, error) + // DeleteStreamLogBucket drops a whole bucket, the unit of reclamation. + DeleteStreamLogBucket(ctx context.Context, request *InternalDeleteStreamLogBucketRequest) error + // GetAllHistoryTreeBranches returns all branches of all trees. // Note that branches may be skipped or duplicated across pages if there are branches created or deleted while // paginating through results. @@ -532,6 +546,44 @@ type ( Events *commonpb.DataBlob } + // InternalAppendStreamLogRequest writes one batch of a stream log. + InternalAppendStreamLogRequest struct { + ShardID int32 + NamespaceID string + CollectionID string + Bucket int64 + StartOffset int64 + NextOffset int64 + Node *commonpb.DataBlob + } + + // InternalReadStreamLogRequest reads the batches covering + // [MinOffset, MaxOffset) from one bucket. + InternalReadStreamLogRequest struct { + ShardID int32 + NamespaceID string + CollectionID string + Bucket int64 + MinOffset int64 + MaxOffset int64 + PageSize int + } + + // InternalReadStreamLogResponse returns batches with the offset each begins + // at, so a caller can drop the part of the first batch it did not ask for. + InternalReadStreamLogResponse struct { + Batches []*commonpb.DataBlob + StartOffsets []int64 + } + + // InternalDeleteStreamLogBucketRequest drops one bucket. + InternalDeleteStreamLogBucketRequest struct { + ShardID int32 + NamespaceID string + CollectionID string + Bucket int64 + } + // InternalAppendHistoryNodesRequest is used to append a batch of history nodes InternalAppendHistoryNodesRequest struct { // The raw branch token diff --git a/common/persistence/persistence_metric_clients.go b/common/persistence/persistence_metric_clients.go index b451295c0d4..fc971a2c850 100644 --- a/common/persistence/persistence_metric_clients.go +++ b/common/persistence/persistence_metric_clients.go @@ -1462,3 +1462,45 @@ func updateErrorMetric(handler metrics.Handler, logger log.Logger, operation str } } } + +// AppendStreamLog writes one batch of a stream log +func (p *executionPersistenceClient) AppendStreamLog( + ctx context.Context, + request *InternalAppendStreamLogRequest, +) (retErr error) { + caller := headers.GetCallerInfo(ctx).CallerName + startTime := time.Now().UTC() + defer func() { + p.healthSignals.Record(CallerSegmentMissing, time.Since(startTime), retErr) + p.recordRequestMetrics(metrics.PersistenceAppendStreamLogScope, caller, time.Since(startTime), retErr) + }() + return p.persistence.AppendStreamLog(ctx, request) +} + +// ReadStreamLog returns the batches covering a range +func (p *executionPersistenceClient) ReadStreamLog( + ctx context.Context, + request *InternalReadStreamLogRequest, +) (_ *InternalReadStreamLogResponse, retErr error) { + caller := headers.GetCallerInfo(ctx).CallerName + startTime := time.Now().UTC() + defer func() { + p.healthSignals.Record(CallerSegmentMissing, time.Since(startTime), retErr) + p.recordRequestMetrics(metrics.PersistenceReadStreamLogScope, caller, time.Since(startTime), retErr) + }() + return p.persistence.ReadStreamLog(ctx, request) +} + +// DeleteStreamLogBucket drops a whole bucket +func (p *executionPersistenceClient) DeleteStreamLogBucket( + ctx context.Context, + request *InternalDeleteStreamLogBucketRequest, +) (retErr error) { + caller := headers.GetCallerInfo(ctx).CallerName + startTime := time.Now().UTC() + defer func() { + p.healthSignals.Record(CallerSegmentMissing, time.Since(startTime), retErr) + p.recordRequestMetrics(metrics.PersistenceDeleteStreamLogBucketScope, caller, time.Since(startTime), retErr) + }() + return p.persistence.DeleteStreamLogBucket(ctx, request) +} diff --git a/common/persistence/persistence_rate_limited_clients.go b/common/persistence/persistence_rate_limited_clients.go index 2cd8fbfb097..0bbefcd0abf 100644 --- a/common/persistence/persistence_rate_limited_clients.go +++ b/common/persistence/persistence_rate_limited_clients.go @@ -1184,3 +1184,36 @@ func ConstructHistoryTaskAPI( ) string { return baseAPI + taskCategory.Name() } + +// AppendStreamLog writes one batch of a stream log +func (p *executionRateLimitedPersistenceClient) AppendStreamLog( + ctx context.Context, + request *InternalAppendStreamLogRequest, +) error { + if err := allow(ctx, "AppendStreamLog", request.ShardID, p.systemRateLimiter, p.namespaceRateLimiter, p.shardRateLimiter); err != nil { + return err + } + return p.persistence.AppendStreamLog(ctx, request) +} + +// ReadStreamLog returns the batches covering a range +func (p *executionRateLimitedPersistenceClient) ReadStreamLog( + ctx context.Context, + request *InternalReadStreamLogRequest, +) (*InternalReadStreamLogResponse, error) { + if err := allow(ctx, "ReadStreamLog", request.ShardID, p.systemRateLimiter, p.namespaceRateLimiter, p.shardRateLimiter); err != nil { + return nil, err + } + return p.persistence.ReadStreamLog(ctx, request) +} + +// DeleteStreamLogBucket drops a whole bucket +func (p *executionRateLimitedPersistenceClient) DeleteStreamLogBucket( + ctx context.Context, + request *InternalDeleteStreamLogBucketRequest, +) error { + if err := allow(ctx, "DeleteStreamLogBucket", request.ShardID, p.systemRateLimiter, p.namespaceRateLimiter, p.shardRateLimiter); err != nil { + return err + } + return p.persistence.DeleteStreamLogBucket(ctx, request) +} diff --git a/common/persistence/persistence_retryable_clients.go b/common/persistence/persistence_retryable_clients.go index 6c413fbdb2d..e4533183e74 100644 --- a/common/persistence/persistence_retryable_clients.go +++ b/common/persistence/persistence_retryable_clients.go @@ -1260,3 +1260,40 @@ func (p *nexusEndpointRetryablePersistenceClient) DeleteNexusEndpoint( } return backoff.ThrottleRetryContext(ctx, op, p.policy, p.isRetryable) } + +// AppendStreamLog writes one batch of a stream log +func (p *executionRetryablePersistenceClient) AppendStreamLog( + ctx context.Context, + request *InternalAppendStreamLogRequest, +) error { + op := func(ctx context.Context) error { + return p.persistence.AppendStreamLog(ctx, request) + } + return backoff.ThrottleRetryContext(ctx, op, p.policy, p.isRetryable) +} + +// ReadStreamLog returns the batches covering a range +func (p *executionRetryablePersistenceClient) ReadStreamLog( + ctx context.Context, + request *InternalReadStreamLogRequest, +) (*InternalReadStreamLogResponse, error) { + var response *InternalReadStreamLogResponse + op := func(ctx context.Context) error { + var err error + response, err = p.persistence.ReadStreamLog(ctx, request) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, p.policy, p.isRetryable) + return response, err +} + +// DeleteStreamLogBucket drops a whole bucket +func (p *executionRetryablePersistenceClient) DeleteStreamLogBucket( + ctx context.Context, + request *InternalDeleteStreamLogBucketRequest, +) error { + op := func(ctx context.Context) error { + return p.persistence.DeleteStreamLogBucket(ctx, request) + } + return backoff.ThrottleRetryContext(ctx, op, p.policy, p.isRetryable) +} diff --git a/common/persistence/sql/history_store.go b/common/persistence/sql/history_store.go index f88582108d8..ae85799bcb0 100644 --- a/common/persistence/sql/history_store.go +++ b/common/persistence/sql/history_store.go @@ -475,3 +475,86 @@ func (m *sqlExecutionStore) GetHistoryTreeContainingBranch( TreeInfos: treeInfos, }, nil } + +// AppendStreamLog writes one batch of a stream log. +// +// A plain upsert, which is the whole point of this table. The key is the offset +// the batch starts at, so a retry of an append replaces the row it wrote before +// rather than competing with it, and there is no transaction-id chain for two +// writers to invert. What a reader may see is decided by the frontier the +// stream component commits, not by anything here. +func (m *sqlExecutionStore) AppendStreamLog( + ctx context.Context, + request *p.InternalAppendStreamLogRequest, +) error { + nsID, err := primitives.ParseUUID(request.NamespaceID) + if err != nil { + return err + } + _, err = m.DB.InsertIntoStreamLog(ctx, &sqlplugin.StreamLogRow{ + ShardID: request.ShardID, + NamespaceID: nsID, + CollectionID: request.CollectionID, + Bucket: request.Bucket, + StartOffset: request.StartOffset, + NextOffset: request.NextOffset, + Data: request.Node.Data, + DataEncoding: request.Node.EncodingType.String(), + }) + if err != nil { + return serviceerror.NewUnavailablef("AppendStreamLog: %v", err) + } + return nil +} + +// ReadStreamLog returns the batches covering the requested range, beginning +// with the batch that contains MinOffset rather than the one that starts at it. +func (m *sqlExecutionStore) ReadStreamLog( + ctx context.Context, + request *p.InternalReadStreamLogRequest, +) (*p.InternalReadStreamLogResponse, error) { + nsID, err := primitives.ParseUUID(request.NamespaceID) + if err != nil { + return nil, err + } + rows, err := m.DB.RangeSelectFromStreamLog(ctx, sqlplugin.StreamLogSelectFilter{ + ShardID: request.ShardID, + NamespaceID: nsID, + CollectionID: request.CollectionID, + Bucket: request.Bucket, + MinOffset: request.MinOffset, + MaxOffset: request.MaxOffset, + PageSize: request.PageSize, + }) + if err != nil { + return nil, serviceerror.NewUnavailablef("ReadStreamLog: %v", err) + } + + resp := &p.InternalReadStreamLogResponse{} + for _, row := range rows { + resp.Batches = append(resp.Batches, p.NewDataBlob(row.Data, row.DataEncoding)) + resp.StartOffsets = append(resp.StartOffsets, row.StartOffset) + } + return resp, nil +} + +// DeleteStreamLogBucket drops a whole bucket. +func (m *sqlExecutionStore) DeleteStreamLogBucket( + ctx context.Context, + request *p.InternalDeleteStreamLogBucketRequest, +) error { + nsID, err := primitives.ParseUUID(request.NamespaceID) + if err != nil { + return err + } + _, err = m.DB.DeleteFromStreamLog(ctx, sqlplugin.StreamLogDeleteFilter{ + ShardID: request.ShardID, + NamespaceID: nsID, + CollectionID: request.CollectionID, + Bucket: request.Bucket, + }) + if err != nil { + return serviceerror.NewUnavailablef("DeleteStreamLogBucket: %v", err) + } + return nil +} diff --git a/common/persistence/sql/sqlplugin/interfaces.go b/common/persistence/sql/sqlplugin/interfaces.go index 662845e9f67..fbff0ab02cf 100644 --- a/common/persistence/sql/sqlplugin/interfaces.go +++ b/common/persistence/sql/sqlplugin/interfaces.go @@ -53,6 +53,7 @@ type ( NexusEndpoints HistoryNode + StreamLog HistoryTree HistoryShard diff --git a/common/persistence/sql/sqlplugin/mysql/stream_log.go b/common/persistence/sql/sqlplugin/mysql/stream_log.go new file mode 100644 index 00000000000..c1bb005192e --- /dev/null +++ b/common/persistence/sql/sqlplugin/mysql/stream_log.go @@ -0,0 +1,64 @@ +package mysql + +import ( + "context" + "database/sql" + + "go.temporal.io/server/common/persistence/sql/sqlplugin" +) + +const ( + // Upsert: the key is the offset a batch starts at, so a rewrite of that + // offset is a retry of the same append rather than a competing one. + insertStreamLogQuery = `INSERT INTO stream_log + (shard_id, namespace_id, collection_id, bucket, start_offset, next_offset, data, data_encoding) + VALUES (:shard_id, :namespace_id, :collection_id, :bucket, :start_offset, :next_offset, :data, :data_encoding) + ON DUPLICATE KEY UPDATE next_offset = VALUES(next_offset), + data = VALUES(data), + data_encoding = VALUES(data_encoding)` + + // The floor subquery finds the batch holding an arbitrary offset: the last + // row starting at or below it. + getStreamLogQuery = `SELECT shard_id, namespace_id, collection_id, bucket, start_offset, next_offset, data, data_encoding ` + + `FROM stream_log ` + + `WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ? ` + + `AND start_offset >= COALESCE((SELECT MAX(start_offset) FROM stream_log ` + + `WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ? AND start_offset <= ?), ?0) ` + + `AND start_offset < ?1 ORDER BY start_offset LIMIT ?2` + + deleteStreamLogQuery = `DELETE FROM stream_log ` + + `WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ?` +) + +func (mdb *db) InsertIntoStreamLog( + ctx context.Context, + row *sqlplugin.StreamLogRow, +) (sql.Result, error) { + return mdb.NamedExecContext(ctx, insertStreamLogQuery, row) +} + +func (mdb *db) RangeSelectFromStreamLog( + ctx context.Context, + filter sqlplugin.StreamLogSelectFilter, +) ([]sqlplugin.StreamLogRow, error) { + var rows []sqlplugin.StreamLogRow + if err := mdb.SelectContext(ctx, &rows, getStreamLogQuery, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket, + filter.MinOffset, + filter.MinOffset, + filter.MaxOffset, + filter.PageSize, + ); err != nil { + return nil, err + } + return rows, nil +} + +func (mdb *db) DeleteFromStreamLog( + ctx context.Context, + filter sqlplugin.StreamLogDeleteFilter, +) (sql.Result, error) { + return mdb.ExecContext(ctx, deleteStreamLogQuery, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket) +} diff --git a/common/persistence/sql/sqlplugin/postgresql/stream_log.go b/common/persistence/sql/sqlplugin/postgresql/stream_log.go new file mode 100644 index 00000000000..acc2ea0a7e1 --- /dev/null +++ b/common/persistence/sql/sqlplugin/postgresql/stream_log.go @@ -0,0 +1,65 @@ +package postgresql + +import ( + "context" + "database/sql" + + "go.temporal.io/server/common/persistence/sql/sqlplugin" +) + +const ( + // Upsert: the key is the offset a batch starts at, so a rewrite of that + // offset is a retry of the same append rather than a competing one. + insertStreamLogQuery = `INSERT INTO stream_log + (shard_id, namespace_id, collection_id, bucket, start_offset, next_offset, data, data_encoding) + VALUES (:shard_id, :namespace_id, :collection_id, :bucket, :start_offset, :next_offset, :data, :data_encoding) + ON CONFLICT (shard_id, namespace_id, collection_id, bucket, start_offset) + DO UPDATE SET next_offset = excluded.next_offset, + data = excluded.data, + data_encoding = excluded.data_encoding` + + // The floor subquery finds the batch holding an arbitrary offset: the last + // row starting at or below it. + getStreamLogQuery = `SELECT shard_id, namespace_id, collection_id, bucket, start_offset, next_offset, data, data_encoding ` + + `FROM stream_log ` + + `WHERE shard_id = $1 AND namespace_id = $2 AND collection_id = $3 AND bucket = $4 ` + + `AND start_offset >= COALESCE((SELECT MAX(start_offset) FROM stream_log ` + + `WHERE shard_id = $5 AND namespace_id = $6 AND collection_id = $7 AND bucket = $8 AND start_offset <= $9), $10) ` + + `AND start_offset < $11 ORDER BY start_offset LIMIT $12` + + deleteStreamLogQuery = `DELETE FROM stream_log ` + + `WHERE shard_id = $1 AND namespace_id = $2 AND collection_id = $3 AND bucket = $4` +) + +func (pdb *db) InsertIntoStreamLog( + ctx context.Context, + row *sqlplugin.StreamLogRow, +) (sql.Result, error) { + return pdb.NamedExecContext(ctx, insertStreamLogQuery, row) +} + +func (pdb *db) RangeSelectFromStreamLog( + ctx context.Context, + filter sqlplugin.StreamLogSelectFilter, +) ([]sqlplugin.StreamLogRow, error) { + var rows []sqlplugin.StreamLogRow + if err := pdb.SelectContext(ctx, &rows, getStreamLogQuery, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket, + filter.MinOffset, + filter.MinOffset, + filter.MaxOffset, + filter.PageSize, + ); err != nil { + return nil, err + } + return rows, nil +} + +func (pdb *db) DeleteFromStreamLog( + ctx context.Context, + filter sqlplugin.StreamLogDeleteFilter, +) (sql.Result, error) { + return pdb.ExecContext(ctx, deleteStreamLogQuery, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket) +} diff --git a/common/persistence/sql/sqlplugin/sqlite/stream_log.go b/common/persistence/sql/sqlplugin/sqlite/stream_log.go new file mode 100644 index 00000000000..7979dd9d6d4 --- /dev/null +++ b/common/persistence/sql/sqlplugin/sqlite/stream_log.go @@ -0,0 +1,72 @@ +package sqlite + +import ( + "context" + "database/sql" + + "go.temporal.io/server/common/persistence/sql/sqlplugin" +) + +const ( + // Upsert, because the key is the offset a batch starts at and a rewrite of + // that offset is a retry of the same append rather than a competing one. + // This is the property the whole substrate exists for: there is nothing to + // order two writers against, so nothing to get wrong. + insertStreamLogQuery = `INSERT INTO stream_log + (shard_id, namespace_id, collection_id, bucket, start_offset, next_offset, data, data_encoding) + VALUES (:shard_id, :namespace_id, :collection_id, :bucket, :start_offset, :next_offset, :data, :data_encoding) + ON CONFLICT (shard_id, namespace_id, collection_id, bucket, start_offset) + DO UPDATE SET next_offset = excluded.next_offset, + data = excluded.data, + data_encoding = excluded.data_encoding` + + // The floor subquery is what lets a reader ask for an arbitrary offset. A + // batch covers a range, so the row holding an offset is the last one that + // starts at or below it, and only the store can find that in one step. + getStreamLogQuery = `SELECT shard_id, namespace_id, collection_id, bucket, start_offset, next_offset, data, data_encoding + FROM stream_log + WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ? + AND start_offset >= COALESCE(( + SELECT MAX(start_offset) FROM stream_log + WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ? + AND start_offset <= ? + ), ?) + AND start_offset < ? + ORDER BY start_offset LIMIT ?` + + deleteStreamLogQuery = `DELETE FROM stream_log + WHERE shard_id = ? AND namespace_id = ? AND collection_id = ? AND bucket = ?` +) + +func (mdb *db) InsertIntoStreamLog( + ctx context.Context, + row *sqlplugin.StreamLogRow, +) (sql.Result, error) { + return mdb.conn.NamedExecContext(ctx, insertStreamLogQuery, row) +} + +func (mdb *db) RangeSelectFromStreamLog( + ctx context.Context, + filter sqlplugin.StreamLogSelectFilter, +) ([]sqlplugin.StreamLogRow, error) { + var rows []sqlplugin.StreamLogRow + if err := mdb.conn.SelectContext(ctx, &rows, getStreamLogQuery, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket, + filter.MinOffset, + filter.MinOffset, + filter.MaxOffset, + filter.PageSize, + ); err != nil { + return nil, err + } + return rows, nil +} + +func (mdb *db) DeleteFromStreamLog( + ctx context.Context, + filter sqlplugin.StreamLogDeleteFilter, +) (sql.Result, error) { + return mdb.conn.ExecContext(ctx, deleteStreamLogQuery, + filter.ShardID, filter.NamespaceID, filter.CollectionID, filter.Bucket) +} diff --git a/common/persistence/sql/sqlplugin/stream_log.go b/common/persistence/sql/sqlplugin/stream_log.go new file mode 100644 index 00000000000..9a8685a36b1 --- /dev/null +++ b/common/persistence/sql/sqlplugin/stream_log.go @@ -0,0 +1,66 @@ +package sqlplugin + +import ( + "context" + "database/sql" + + "go.temporal.io/server/common/primitives" +) + +type ( + // StreamLogRow is one appended batch, keyed by the offset its first message + // landed at. + // + // Keying by offset rather than by an opaque node id is what makes a write + // idempotent: a retry of the same append addresses the same row and + // replaces it. There is no chain to order two writers against, and no way + // for an uncommitted write to outrank a later one. The frontier the stream + // component holds is the only thing that decides what a reader may see. + StreamLogRow struct { + ShardID int32 + NamespaceID primitives.UUID + CollectionID string + Bucket int64 + StartOffset int64 + // One past the last offset this batch covers, so a reader knows whether + // the row holds the offset it asked for without decoding the blob. + // + // Not called end_offset: the schema loader scans for the SQL keyword + // `END` and a column beginning with it fails to parse. + NextOffset int64 + Data []byte + DataEncoding string + } + + // StreamLogSelectFilter reads the batches covering [MinOffset, MaxOffset). + // + // The read begins at the batch containing MinOffset, which is the greatest + // start_offset at or below it, not at MinOffset itself. A store resolves + // that itself; the caller does not have to guess how far back a batch may + // have begun. + StreamLogSelectFilter struct { + ShardID int32 + NamespaceID primitives.UUID + CollectionID string + Bucket int64 + MinOffset int64 + MaxOffset int64 + PageSize int + } + + // StreamLogDeleteFilter drops a whole bucket, which is the unit of + // reclamation. + StreamLogDeleteFilter struct { + ShardID int32 + NamespaceID primitives.UUID + CollectionID string + Bucket int64 + } + + // StreamLog is the SQL persistence interface for stream log batches. + StreamLog interface { + InsertIntoStreamLog(ctx context.Context, row *StreamLogRow) (sql.Result, error) + RangeSelectFromStreamLog(ctx context.Context, filter StreamLogSelectFilter) ([]StreamLogRow, error) + DeleteFromStreamLog(ctx context.Context, filter StreamLogDeleteFilter) (sql.Result, error) + } +) diff --git a/common/persistence/tests/history_store_stream_log.go b/common/persistence/tests/history_store_stream_log.go new file mode 100644 index 00000000000..9b59c57f242 --- /dev/null +++ b/common/persistence/tests/history_store_stream_log.go @@ -0,0 +1,385 @@ +package tests + +import ( + "time" + + "github.com/google/uuid" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm/lib/stream" + "go.temporal.io/server/common" + p "go.temporal.io/server/common/persistence" +) + +// Tests covering the history-node store's behaviour as a general append-only +// log, addressed by a branch whose tree ID is not a workflow run ID. +// +// These exist because a stream primitive reuses this store directly rather than +// standing up a parallel one. Two properties carry that decision: the node blob +// is never interpreted, and a stale node left behind by a shrinking retry is +// rejected on read even once the visibility frontier has moved past it. Neither +// was covered before, and the second is the one that decides whether the reuse +// is sound at all. + +// newLogBranch mints a branch whose tree ID is a fresh UUID rather than a run ID. +func (s *HistoryEventsSuite) newLogBranch() []byte { + branchID := uuid.NewString() + branchToken, err := s.store.GetHistoryBranchUtil().NewHistoryBranch( + uuid.NewString(), + uuid.NewString(), + uuid.NewString(), + uuid.NewString(), // tree ID: not a run ID + &branchID, + []*persistencespb.HistoryBranchRange{}, + time.Duration(0), + time.Duration(0), + time.Duration(0), + ) + s.NoError(err) + return branchToken +} + +func (s *HistoryEventsSuite) appendLogNode( + shardID int32, + branchToken []byte, + nodeID int64, + txnID int64, + prevTxnID int64, + blob *commonpb.DataBlob, +) { + _, err := s.store.AppendRawHistoryNodes(s.Ctx, &p.AppendRawHistoryNodesRequest{ + ShardID: shardID, + BranchToken: branchToken, + NodeID: nodeID, + TransactionID: txnID, + PrevTransactionID: prevTxnID, + IsNewBranch: nodeID == common.FirstEventID, + Info: "", + History: blob, + }) + s.NoError(err) +} + +func (s *HistoryEventsSuite) listRawLogNodes( + shardID int32, + branchToken []byte, + minNodeID int64, + maxNodeID int64, +) ([]*commonpb.DataBlob, []int64) { + var token []byte + var blobs []*commonpb.DataBlob + var nodeIDs []int64 + for doContinue := true; doContinue; doContinue = len(token) > 0 { + resp, err := s.store.ReadRawHistoryBranch(s.Ctx, &p.ReadHistoryBranchRequest{ + ShardID: shardID, + BranchToken: branchToken, + MinEventID: minNodeID, + MaxEventID: maxNodeID, + PageSize: 1, + NextPageToken: token, + }) + s.NoError(err) + token = resp.NextPageToken + blobs = append(blobs, resp.HistoryEventBlobs...) + nodeIDs = append(nodeIDs, resp.NodeIDs...) + } + return blobs, nodeIDs +} + +func (s *HistoryEventsSuite) eventIDsOf(events []*historypb.HistoryEvent) []int64 { + ids := make([]int64, len(events)) + for i, e := range events { + ids[i] = e.EventId + } + return ids +} + +// TestStreamLogBlobIsOpaque checks the store round-trips a node blob it cannot +// parse. A stream keeps application payloads here, so any attempt to interpret +// the bytes as history events would reject them. +func (s *HistoryEventsSuite) TestStreamLogBlobIsOpaque() { + branchToken := s.newLogBranch() + + payload := []byte("not a History proto \x00\x01\x02 arbitrary stream bytes") + blob := &commonpb.DataBlob{ + EncodingType: enumspb.ENCODING_TYPE_PROTO3, + Data: payload, + } + + s.appendLogNode(s.ShardID, branchToken, common.FirstEventID, 100, 0, blob) + + blobs, nodeIDs := s.listRawLogNodes(s.ShardID, branchToken, common.FirstEventID, common.FirstEventID+1) + s.Len(blobs, 1) + s.Equal([]int64{common.FirstEventID}, nodeIDs) + s.Equal(payload, blobs[0].Data) +} + +// TestStreamLogShrinkingRetryDropsStaleNode covers the case where a retry writes +// fewer nodes than the attempt it replaces, leaving a stale node past the +// retry's extent. Once later appends move the frontier beyond it, clipping the +// read range no longer hides it, so the store's transaction-ID chain has to +// reject it. +// +// Layout: node 1 commits, then a failed attempt writes nodes 11 and 13, then a +// smaller retry writes node 11 alone, then node 12 commits. Node 13 is stale +// and now sits below the frontier. +func (s *HistoryEventsSuite) TestStreamLogShrinkingRetryDropsStaleNode() { + branchToken := s.newLogBranch() + + committed := s.newHistoryEvents([]int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 100, 0) + s.appendRawHistoryBatches(s.ShardID, branchToken, committed) + + // Attempt that fails after writing both nodes; the frontier never advances. + staleFirst := s.newHistoryEvents([]int64{11, 12}, 200, 100) + s.appendRawHistoryBatches(s.ShardID, branchToken, staleFirst) + staleSecond := s.newHistoryEvents([]int64{13, 14}, 201, 200) + s.appendRawHistoryBatches(s.ShardID, branchToken, staleSecond) + + // Retry carries fewer messages, so it covers node 11 only. + retry := s.newHistoryEvents([]int64{11}, 300, 100) + s.appendRawHistoryBatches(s.ShardID, branchToken, retry) + + // Next append moves the frontier past the stale node at 13. + next := s.newHistoryEvents([]int64{12, 13, 14, 15}, 400, 300) + s.appendRawHistoryBatches(s.ShardID, branchToken, next) + + events := s.listHistoryEvents(s.ShardID, branchToken, common.FirstEventID, 16) + s.Equal( + []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + s.eventIDsOf(events), + "stale node 13 from the abandoned attempt must not be returned", + ) + + // The raw path carries no contiguity check of its own, so for a caller that + // never parses the blob the transaction-ID chain is the only thing standing + // between it and the stale node. + _, nodeIDs := s.listRawLogNodes(s.ShardID, branchToken, common.FirstEventID, 16) + s.Equal([]int64{1, 11, 12}, nodeIDs, "raw reads must drop the stale node too") +} + +// TestStreamLogTrimReclaimsStaleNodes checks that trimming against the committed +// frontier reclaims stale nodes without disturbing the valid chain. This is what +// lets a stream clean up after a failed append promptly instead of waiting on a +// background scavenger. +func (s *HistoryEventsSuite) TestStreamLogTrimReclaimsStaleNodes() { + branchToken := s.newLogBranch() + + committed := s.newHistoryEvents([]int64{1, 2, 3}, 100, 0) + s.appendRawHistoryBatches(s.ShardID, branchToken, committed) + + stale := s.newHistoryEvents([]int64{4, 5}, 200, 100) + s.appendRawHistoryBatches(s.ShardID, branchToken, stale) + + retry := s.newHistoryEvents([]int64{4}, 300, 100) + s.appendRawHistoryBatches(s.ShardID, branchToken, retry) + + s.trimHistoryBranch(s.ShardID, branchToken, 4, 300) + + events := s.listHistoryEvents(s.ShardID, branchToken, common.FirstEventID, 5) + s.Equal([]int64{1, 2, 3, 4}, s.eventIDsOf(events)) +} + +// The bucketing scheme in chasm/lib/stream splits a stream across one tree per +// fixed-size offset range, so a storage partition cannot grow with the stream. +// That also means a read restarts the transaction chain at every bucket, and +// the claim that this is still safe is the one worth testing rather than +// asserting: it is the same class of claim that turned out to be wrong about +// partition growth in the first place. + +func (s *HistoryEventsSuite) streamAppend( + collectionID string, + bucketSize int64, + firstOffset int64, + count int64, + body string, +) { + blob := &commonpb.DataBlob{ + EncodingType: enumspb.ENCODING_TYPE_PROTO3, + Data: []byte(body), + } + err := stream.WriteAppend(s.Ctx, s.store, s.ShardID, testStreamNamespaceID, collectionID, stream.LogAppend{ + Bucket: stream.BucketOf(firstOffset, bucketSize), + StartOffset: firstOffset, + NextOffset: firstOffset + count, + Blob: blob, + }) + s.NoError(err) +} + +func (s *HistoryEventsSuite) streamRead( + collectionID string, + bucketSize int64, + from int64, + to int64, +) []string { + blobs, _, err := stream.ReadRange( + s.Ctx, s.store, s.ShardID, testStreamNamespaceID, collectionID, bucketSize, from, to, 0) + s.NoError(err) + out := make([]string, len(blobs)) + for i, b := range blobs { + out[i] = string(b.Data) + } + return out +} + +const testStreamNamespaceID = "0b9d2c3a-1f4e-4a7b-9c8d-5e6f70819203" + +// TestStreamLogBucketedReadSpansTrees checks that a range read stitches buckets +// together, since each bucket is a separate tree and a caller only ever sees +// global offsets. +func (s *HistoryEventsSuite) TestStreamLogBucketedReadSpansTrees() { + collectionID := uuid.NewString() + const bucketSize = 4 + + s.streamAppend(collectionID, bucketSize, 0, 4, "bucket0") + s.streamAppend(collectionID, bucketSize, 4, 4, "bucket1") + s.streamAppend(collectionID, bucketSize, 8, 2, "bucket2") + + s.Equal([]string{"bucket0", "bucket1", "bucket2"}, s.streamRead(collectionID, bucketSize, 0, 10)) + s.Equal([]string{"bucket1"}, s.streamRead(collectionID, bucketSize, 4, 8)) +} + +// TestStreamLogBucketBoundaryDropsStaleNode is the bucket-aware form of +// TestStreamLogShrinkingRetryDropsStaleNode. An abandoned append straddles a +// bucket boundary, so the stale node lands in a tree whose chain a reader +// starts fresh. It must still be rejected once the frontier moves past it. +func (s *HistoryEventsSuite) TestStreamLogBucketBoundaryDropsStaleNode() { + collectionID := uuid.NewString() + const bucketSize = 4 + + s.streamAppend(collectionID, bucketSize, 0, 3, "committed") + + // Abandoned attempt: tail of bucket 0 plus the head of bucket 1. + s.streamAppend(collectionID, bucketSize, 3, 1, "stale-bucket0") + s.streamAppend(collectionID, bucketSize, 4, 2, "stale-bucket1") + + // Retry covers only bucket 0, so the bucket 1 node is orphaned. + s.streamAppend(collectionID, bucketSize, 3, 1, "retry") + + // A later append reaches into bucket 1, moving the frontier past the orphan. + s.streamAppend(collectionID, bucketSize, 4, 2, "real-bucket1") + + s.Equal( + []string{"committed", "retry", "real-bucket1"}, + s.streamRead(collectionID, bucketSize, 0, 6), + "the orphaned bucket 1 node must not surface once the frontier passes it", + ) +} + +// TestStreamLogOrphanFromAnotherSequenceShadowsLaterWrites is the failure the +// prototype shipped with: two producers numbering from two unrelated sequences. +// +// The chain rule keeps the highest transaction id it has seen and drops +// everything below it, which orders contested nodes correctly only while every +// writer draws from one sequence. A node left behind by a write that never +// committed still counts, because the rule reads storage and not the frontier. +// So an uncommitted node numbered from a sequence that runs ahead hides every +// later write numbered from the one that runs behind, and the reader is not +// told: it simply stops seeing new messages. +// +// The fix is one sequence per shard for every producer. This test pins the +// behaviour that made the bug invisible, so a second sequence cannot come back. +func (s *HistoryEventsSuite) TestStreamLogOrphanFromAnotherSequenceShadowsLaterWrites() { + branchToken := s.newLogBranch() + + // A committed append, numbered from the sequence the workflow used. + first := s.newHistoryEvents([]int64{1, 2}, 32, 0) + s.appendRawHistoryBatches(s.ShardID, branchToken, first) + + // A producer outside the workflow writes its node and then fails to commit, + // so the frontier never covers it. Its id comes from the shard generator and + // is far above anything the workflow task path produces. + orphan := s.newHistoryEvents([]int64{3, 4}, 5000, 32) + s.appendRawHistoryBatches(s.ShardID, branchToken, orphan) + + // Two more committed appends from the workflow. Both are numbered below the + // orphan, which is the whole problem. + second := s.newHistoryEvents([]int64{3, 4}, 33, 32) + s.appendRawHistoryBatches(s.ShardID, branchToken, second) + third := s.newHistoryEvents([]int64{5, 6}, 34, 33) + s.appendRawHistoryBatches(s.ShardID, branchToken, third) + + events := s.listHistoryEvents(s.ShardID, branchToken, common.FirstEventID, 7) + s.Equal( + []int64{1, 2, 3, 4}, + s.eventIDsOf(events), + "the orphan is served and both committed appends after it are dropped, "+ + "which is why a second transaction-id sequence loses data", + ) + + // The same log, written the way the fix writes it: every producer numbering + // from the shard, so the orphan is superseded rather than dominant. + fixed := s.newLogBranch() + s.appendRawHistoryBatches(s.ShardID, fixed, s.newHistoryEvents([]int64{1, 2}, 5001, 0)) + s.appendRawHistoryBatches(s.ShardID, fixed, s.newHistoryEvents([]int64{3, 4}, 5002, 5001)) + s.appendRawHistoryBatches(s.ShardID, fixed, s.newHistoryEvents([]int64{3, 4}, 5003, 5001)) + s.appendRawHistoryBatches(s.ShardID, fixed, s.newHistoryEvents([]int64{5, 6}, 5004, 5003)) + + events = s.listHistoryEvents(s.ShardID, fixed, common.FirstEventID, 7) + s.Equal( + []int64{1, 2, 3, 4, 5, 6}, + s.eventIDsOf(events), + "one sequence per shard: the uncommitted node is superseded and nothing is lost", + ) +} + +// TestStreamLogAppendIsIdempotentByOffset is the property the dedicated facet +// exists for. +// +// A row is keyed by the offset its batch starts at, so a retry of an append +// addresses the row it wrote before and replaces it. There is no chain to order +// two writers against, so there is nothing for an uncommitted write to outrank, +// which is the failure the previous substrate had. +func (s *HistoryEventsSuite) TestStreamLogAppendIsIdempotentByOffset() { + const collectionID = "idempotent-by-offset" + const bucketSize = 100 + + s.streamAppend(collectionID, bucketSize, 0, 2, "first") + + // An attempt that wrote and never committed, at offsets the next attempt + // will reuse. On the old substrate this could outrank what followed. + s.streamAppend(collectionID, bucketSize, 2, 3, "abandoned") + + // The retry, covering fewer offsets from the same start. + s.streamAppend(collectionID, bucketSize, 2, 1, "retry") + + // Whatever wrote last at that offset is what is there, and nothing before + // or after it was disturbed. + s.streamAppend(collectionID, bucketSize, 3, 1, "after") + + s.Equal( + []string{"first", "retry", "after"}, + s.streamRead(collectionID, bucketSize, 0, 4), + "a rewrite at an offset replaces that batch and shadows nothing", + ) +} + +// TestStreamLogReadFindsTheBatchHoldingAnOffset checks that a read starting +// inside a batch gets the batch containing it. +// +// The caller asks for an offset, not for a batch. Only the store can find the +// row holding it, which it does in one indexed lookup because the key is the +// offset a batch starts at. The previous substrate could not, and compensated +// by reading a whole batch's worth of rows backwards on every read. +func (s *HistoryEventsSuite) TestStreamLogReadFindsTheBatchHoldingAnOffset() { + const collectionID = "mid-batch-read" + const bucketSize = 100 + + s.streamAppend(collectionID, bucketSize, 0, 10, "wide") + s.streamAppend(collectionID, bucketSize, 10, 1, "narrow") + + // Offset 4 sits inside the first batch, which starts at 0. + s.Equal( + []string{"wide", "narrow"}, + s.streamRead(collectionID, bucketSize, 4, 11), + "a read landing mid-batch must be served the batch that holds it", + ) + + // And a read starting exactly on a boundary gets only what follows. + s.Equal( + []string{"narrow"}, + s.streamRead(collectionID, bucketSize, 10, 11), + ) +} diff --git a/common/util.go b/common/util.go index 87e02bbc63a..f67c0daa7e7 100644 --- a/common/util.go +++ b/common/util.go @@ -544,6 +544,7 @@ func CreateMatchingPollWorkflowTaskQueueResponse(historyResponse *historyservice StartedTime: historyResponse.StartedTime, Queries: historyResponse.Queries, Messages: historyResponse.Messages, + StreamSlices: historyResponse.StreamSlices, History: historyResponse.History, NextPageToken: historyResponse.NextPageToken, RawHistory: historyResponse.RawHistoryBytes, diff --git a/design-comparison.md b/design-comparison.md new file mode 100644 index 00000000000..87cfa3277b8 --- /dev/null +++ b/design-comparison.md @@ -0,0 +1,290 @@ +# Native Streams: Design Comparison + +| | | +|---|---| +| Status | Draft for review | +| Ticket | [AI-198](https://temporalio.atlassian.net/browse/AI-198) | +| Author | Moe Dashti | +| Date | 2026-08-23 | +| Canonical option list | [Native Streaming: Options Discussed](https://app.notion.com/p/3b28fc567738812f8c67ca3ebdf9ce38) in Notion | +| Compares | `streaming-high-level-design.md` + `streaming-detailed-design.md` against the implemented prototypes | + +**The Notion page is the single source of truth for the option comparison.** This document is narrower and does not duplicate it: it compares our design against the two prototypes that exist as running code, and against Option 7 specifically, since Option 7 targets the same problem as our Path C. + +Our design was written clean-room, before reading either prototype. Sections 8 and 9 are the actionable part. + +Where our design sits in the canonical list: it is closest to **Option 5** (CHASM-backed append-only collection) but differs in two ways that matter. It does not depend on the CHASM collection primitive, because payload bytes never enter the CHASM tree, and it carries a bucket-2 mechanism that differs from Option 7's. It is arguably a separate option and could be added to the Notion page as one. + +--- + +## 1. What is being compared + +**Ours (server-side log).** The stream is a CHASM component holding only a frontier; payload bytes live in the stream's own `history_node` branch. Appends do not schedule workflow tasks. Readers own their cursor. In-workflow consumption records the consumed offset range and attaches the bytes out of band. + +**Option 7 / Max's prototype (external store, long-lived workflow task).** These are the same thing, which is worth stating plainly because the canonical page lists Option 7 with status "prototype intended". **It already exists**: `mfateev/sdk-python` branch `task/python-sdk-streaming`, roughly 9,500 lines under `temporalio/contrib/external_workflow_streams/`, plus about 40 ADRs and a conformance suite in `mfateev/sdk-core` `arch_docs/streaming-poc-docs/`. Its own overview describes exactly Option 7: "while a Workflow Task is open, the SDK runtime reads the external stream directly. No Temporal Server changes are required." The ADRs contain findings the one-paragraph summary does not carry; see §6. + +**Johann's prototype (server-side facet).** `temporalio/internal-ai-prototypes` branch `2026/05/native-streams`, server code at `origin/native-streams-prototype`, roughly 9,890 lines across 67 files. A CHASM `Stream` component plus a new `stream_segments` persistence facet on four backends, a 3-step cross-facet commit protocol, and a TLC-verified TLA+ spec. Python client. No workflow integration. This is the implementation of Option 5, minus the shared collection primitive. + +--- + +## 2. Side by side + +| | Ours | Max's | Johann's | +|---|---|---|---| +| Where payload bytes live | existing `history_node` branch | external store (Redis) | new `stream_segments` table | +| Server changes | CHASM lib + one framework hook | none | CHASM lib + new persistence facet + schema on 4 backends | +| SDK core changes | none required for the client path | required (`ReplayExternalStreams` activation job, new commands) | none | +| New DB schema | none | none (external) | 2 tables x 4 backends | +| Durability owner | Temporal | the external store | Temporal | +| Replication | rides history-node replication | not covered | deferred, facet is replication-friendly | +| Exactly-once write | server-enforced, `(producerId, seq)` | backend adapter idempotency, explicitly a non-goal | server-enforced, `(publisher_id, sequence)` | +| Commit protocol | none beyond append-then-advance | n/a | 3-step cross-facet, TLA+ verified | +| LWTs per publish (Cassandra) | 1 | 0 (no Temporal write) | 2, or 2/G with group commit | +| Offset to storage lookup | node-ID range read, no index | provider offsets | `SegmentIndex` on the chasm node, binary search, close-time offload | +| Client tail | long-poll via `chasm.PollComponent` | direct backend read plus wake Signal | not built (`ReadRange` is non-blocking) | +| Subscriber limit | memory-bound | backend-bound | bounded outstanding window per subscription | +| Workflow publish | command inside the existing WFT commit | producer handle to the external store | designed, not built | +| Workflow consume | offset range in history, bytes out of band | marker annotation, bytes from the backend | items pushed **as signals into the consumer's history** | +| Consumer-side dedup burden | none | none (ranges are recorded) | **on the workflow handler**, at-least-once | +| Maturity | design only | working, heavily tested, ~40 ADRs | working primitive, TLA+ spec, no workflow integration | + +--- + +## 3. Where all three agree + +Worth stating, because the convergence is evidence the shape is right: + +- Items must not go into workflow history one per event. +- Offsets are the addressing model, and the reader carries its own cursor. +- Publishing must not wake the owning workflow. +- The stream outlives the workflow task that produced it, and close is distinct from delete. +- Rewinds on retry or reset are surfaced to the application rather than hidden. +- Topics, not predicates, for filtering. + +Most significantly, **Max and we independently arrived at the same replay model**: record the consumed offset range in history and re-read the bytes from an immutable log. He calls it a marker annotation, we call it a cursor event. That two designs built without knowledge of each other landed on the same mechanism is the strongest single signal in this comparison. + +--- + +## 4. Option 7 / Max's prototype + +### What it gets right + +- **No server changes.** It could ship on today's server, which no other option can claim. +- **The replay model is correct and battle-tested.** Around 40 ADRs, each pairing a rule with the failure that occurs without it, and a conformance suite for backends. +- **It found the hard cases.** Cursor as a position boundary rather than a record identity (ADR-002), because a consumer parked at the tail has no next-record ID to persist. Progress as an observation delta emitted on every completion path (ADR-005), because a subscription that observed nothing still has to record that it observed nothing. Activation segmentation reproduced rather than collapsed (ADR-018), because `wait_condition` fires once per activation and collapsing k drains into one changes when conditions fire. +- **Cost claim is sharp**: history event cost scales with consumption batches and idle-to-active transitions, not item count, and marker bytes are capped by a byte budget that forces rollover instead of growing a marker. + +### Where it does not fit our goal + +- **Durability moves outside Temporal.** This is the objection the 7/23 options doc already recorded: if workflows replicate and the external stream does not, the guarantees become inconsistent. Structural immutability of the backend has to be asserted at registration time (ADR-003) rather than being true by construction, and the doc states plainly that a provider which silently violates it delivers altered bytes on replay with no error raised. +- **It needs an external system.** That is the Redis workaround the feature exists to remove, made official. The PRD lists extra infrastructure, split observability, and a second failure domain as the reasons the workaround is not sufficient. +- **Exactly-once producer execution is an explicit non-goal**, delegated to backend adapter idempotency. +- **The replay machinery is inherently complex** because the SDK reads the backend continuously while a workflow task is open. The boundary of what was observed is not a natural artifact of anything, so it has to be reconstructed: runs, segments, per-segment end reasons, sparse control positions, and a byte budget. +- Requires sdk-core changes, so "client side" means "no server changes", not "no protocol changes". + +### Option 7 against our Path C + +Both target bucket 2. This is now the live disagreement, so it is worth being precise rather than summarising. + +| | Option 7 | Ours (Path C) | +|---|---|---| +| Where payload lives | external store (Redis, Postgres) | Temporal-owned log | +| Server changes | none, SDK-only | CHASM lib, transaction hook, api-go | +| Workflow task shape | held open while reading, completes on idle (~1-2s) or explicit flush | normal duration, one slice per task | +| Recorded for replay | "stream-empty" markers at task completion | delivered range on every `WorkflowTaskCompleted`, including empty | +| Failure mid-stream | application-level reset protocol, visible to users | nothing to do; the log is immutable and the cursor is recorded | +| Exactly-once | not provided; idempotent append via the backend adapter | server-enforced | +| Cost per LLM invocation | about 1 signal + 1 marker, data cost paid to Redis | 0 events, data cost paid to Temporal storage | +| Time to ship | fastest of any option | slowest | + +The honest summary: **Option 7 is cheaper and faster to ship because the data never enters Temporal at all.** That is the same trade as Option 1 against Option 5, applied to bucket 2. It is not a trade we can argue our way out of, and pretending our design wins on cost would be wrong. + +Where I think Option 7 needs scrutiny, phrased as questions rather than objections: + +1. **Long-lived workflow tasks against the workflow-task timeout.** Holding a task open for the length of an LLM response means the task lives for seconds to minutes. Gap K3 in the 8/10 WTAL list is exactly "you still can't raise a WFT timeout; need to not kill a WFT that's still making progress", and it is marked *needs investigation*. Is Option 7 gated on K3? Separately, while a task is open, incoming signals buffer, and `MaximumBufferedEventsBatch` defaults to 100 with a 2MB cap (`common/dynamicconfig/constants.go:2610`), after which the task is force-failed. A workflow taking sustained signal traffic during a long read would hit that. +2. **Worker slot occupancy.** A held task pins a worker slot for the LLM's duration, which changes worker sizing in a way per-task execution does not. Worth measuring, not assuming. +3. **Is the replay analysis complete?** The summary says the only non-determinism risk is empty-on-original, non-empty-on-replay. Max's own prototype found more: ADR-018 concludes that replay must reproduce **activation segmentation**, the number of drains, not just the record order, because `wait_condition` predicates evaluate once per activation and collapsing k drains into one changes when conditions fire. ADR-005 adds that the first observation of a subscription must carry provider identity and resolved start boundary even when nothing was delivered. Those are real and already solved in the branch, but they are not in the summary, so the summary understates the replay work. +4. **Durability of the wake-up.** The notes say the signal must be durable, then float a rejected update as a lighter-weight substitute. A rejected update leaves no history event. If the workflow is not running when the stream opens, what guarantees the wake is not lost? + +And where Path C is genuinely simpler, for a structural reason rather than a cleverness one: our delivery boundary **is** the workflow task boundary, and that boundary is already durable. So there is no segmentation to reproduce, no idle heuristic to tune, and no reset protocol, because a failed task leaves an immutable log and a cursor that was never advanced. Option 7 has to reconstruct all three because it reads continuously inside a task, which is also precisely what buys it its lower latency and lower cost. + +### They are not mutually exclusive + +Worth putting on the table before the comparison hardens into a choice. + +Option 7 is a **consumption mechanism** over a pluggable store whose contract is append plus read-from-offset. Our design is a **store** with that same contract, plus a consumption mechanism. If the SDK-side reading model is written provider-agnostic, then a Temporal-native stream is simply one more provider behind it. + +That suggests a sequence rather than a fork: ship Option 7's SDK model with Redis and Postgres bindings first, because it is fastest and the urgency is real, and add a Temporal-backed provider for customers who need durability and replication inside Temporal. Customers who are happy running Redis get unblocked now; customers who cannot run a second stateful system get an answer later without a second API. + +The cost of not doing this is two stream APIs. + +### What we take + +1. **Record the range even when it is empty**, and carry the resolved start offset on first observation (ADR-005). Our Path C as written only wrote an event when items were delivered, which leaves replay unable to reproduce a task where the subscription observed nothing. That is a correctness hole, and this is the fix. +2. **A per-producer write fence distinct from closing the stream** (ADR-040). A producer can declare it is done without ending the stream for other producers. +3. **Bound delivery by record count as well as bytes** (ADR-026 with ADR-007). +4. **Attached-stream identity keyed on the first execution run ID**, so it is stable across a continue-as-new chain and does not collide after workflow ID reuse. +5. **Missing data on replay blocks rather than fails** (ADR-014): surface a retryable workflow task failure, not a nondeterminism error. +6. **An explicit flush control message.** Option 7 ends a read when the stream is idle for a second or two, or when the writer sends an explicit flush. Our design bounds a slice by size and count but gives the producer no way to say "the turn is finished, deliver now". Without it a consumer waits on a timeout it should not need. Cheap to add and it removes a tuning knob rather than adding one. +7. **A per-topic sequence number alongside the global offset.** Option 7 multiplexes logical streams over one physical stream with per-stream IDs so one tool call can reset without disturbing the others. Our offsets are global across topics, which is right for ordering but leaves a consumer no cheap way to reason about one topic's progress. + +### What we do not take + +The external backend itself. It is a legitimate product option for customers who want to pay a different price for volume, and the 1-pager says so. It is not this design. Our read API is offset plus long-poll, which is the same shape a backend adapter would expose, so the two can sit behind one client API later if we choose. + +--- + +## 5. Johann's approach + +### What it gets right + +- **The durability story is ours too**, and it is the right one: every item a subscriber sees is already persisted by the history server. +- **It is the most rigorous artifact of the three on the write path.** A TLA+ spec that TLC exhausts in about 15 seconds, with every action mapped to a Go function in `spec/SPEC_TO_CODE.md`. +- **It settled a lot of product surface** we would otherwise re-litigate: multi-topic streams with subscribe-time filtering, `ListStreams` for operators, owner link on business ID so it survives continue-as-new, close-not-delete on owner completion, no invented per-stream TTL, and end-to-end codec so the server never sees plaintext. +- **Positioning is explicit and honest**: many modest-volume streams, not a firehose, with per-stream throughput bounded by the CHASM transition rate on a single execution. +- **Inline retention truncation** at the end of each successful publish transition, with a consumer pin, rather than a separate sweeper. + +### Where we differ, and why + +**The persistence choice is the crux.** Johann's `persistence-abi.html` evaluates four approaches and picks C, a dedicated append-log facet. Its own stated rationale is that this "mirrors workflow history events, which the team already operates at scale". Our design takes that observation one step further: rather than building a facet that mirrors `history_node`, use `history_node`. + +The doc anticipates this and rejects it in one line: history's pattern is "inspiration but not a drop-in template" because "history's 12-hour scavenger and 60-day min-age work because history orphans are rare; streams generate them routinely (publish retries, truncate races, backpressure) and need a prepare / write / commit protocol with explicit visibility frontier and a 1-5 minute sweeper." + +That objection is about garbage, not correctness, and it has an answer that was not considered: + +- **Correctness is already handled by the store.** `filterHistoryNodes` (`common/persistence/history_manager.go:1039-1073`) does not merely prefer the higher transaction ID for the same node ID. It requires transaction IDs to be non-decreasing along the node chain and drops any node whose transaction ID went backwards. Its own comment says it: "event batches with larger node ID -> batch with lower transaction ID is invalid (happens before)". So an orphan beyond a retry's extent is dropped on read, not just an orphan at the same offset. Combined with clipping reads to `HeadOffset`, no reader can observe an orphan. +- **Garbage collection already has a purpose-built tool.** `TrimHistoryBranch(BranchToken, NodeID, TransactionID)` takes a known-valid frontier and removes everything off the valid chain. The Stream component holds exactly that frontier. So the sweep is a cheap pure task fired after a failed append, not a 12-hour scavenger. The premise that we would inherit history's coarse cleanup cadence does not hold. + +With those two, the prepare/write/commit protocol is not needed, and the design collapses to append-then-advance, which is the pair Temporal has run in production for a decade. + +The consequences are concrete: + +| | Johann's C | Ours | +|---|---|---| +| New tables | `stream_segments` + `stream_sealed_indexes`, 4 backends | none | +| Schema migrations | Cassandra 1.14, MySQL/PG 1.20, SQLite 0.12 | none | +| Commit protocol | 3 steps, TLA+ needed to trust it | append-then-advance | +| Cassandra LWTs per publish | 2 | 1 | +| Offset lookup structure | `SegmentIndex` on the chasm node, binary search, plus a close-time offload to a sibling table when it grows | none; the branch token plus a node-ID range read | +| Tentative-row sweeper | 1 to 5 minutes, required | pure task on failure, opportunistic | +| Replication | new facet needs its own story | rides history-node replication | + +Halving the LWT count matters more than it looks. Johann's own `spec/perf-back-of-envelope.md` identifies the per-partition LWT rate (budgeted at 100/sec) as the per-stream ceiling, and concludes that group commit is "load-bearing, not optional" to hit the 300 items/sec target. Starting at one LWT per publish rather than two doubles the base before group commit is applied, and group commit applies to our design equally. + +**The second difference is in-workflow consumption.** Johann's decision D9 pushes matching items to the workflow **as signals or updates on the consumer's own history**, at-least-once, with handlers deduplicating on item offset. That keeps items out of the *publisher's* history but puts them into the *consumer's*, which is the same cost in a different place, and it puts the dedup burden on user code. The Bellevue session later set the opposite requirement: "the end user never handles delivery deduplication". D9 predates that session, so this is a case of the session superseding an earlier decision rather than a disagreement. + +Our Path C records only the offset range and attaches the bytes to the workflow task response out of band, so nothing per-item enters the consumer's history and delivery is exactly-once by construction. It also needs no subscription state machine, no three-phase delivery protocol, and no outstanding-bytes arm step, because delivery is pulled at task-build time rather than pushed. + +There is a further advantage that only becomes visible next to Max's design. Because our delivery boundary is the workflow task boundary, and that boundary is already in history, we do not have the segmentation problem ADR-018 solves. One slice arrives per task, the SDK drains it once, and replay drains the identical slice once. No runs, no segments, no per-segment end reasons. + +### What we take + +1. **Group commit.** Coalesce concurrent appends to one stream into a single CHASM transition. This is the difference between meeting and missing the throughput target on Cassandra. +2. **Explicit positioning and a stated per-stream ceiling**, rather than letting readers assume a firehose. +3. **Inline retention truncation** at the end of a publish transition, with a consumer pin, instead of a sweeper. +4. **Multi-topic streams with subscribe-time filtering**, and `ListStreams` for operators. +5. **Owner link on business ID, close-not-delete on owner completion**, and no invented per-stream TTL beyond the lifecycle policy the Bellevue session asked for. +6. **Say out loud that the server never sees plaintext.** Our blobs are opaque already, so the codec property is free; it just was not written down. + +--- + +## 6. The Codex design + +A fourth design, at `github.com/temporalio/temporal/streaming-{high-level,detailed}-design.md`. It is a design document, not running code, and it is by far the closest to ours: bounded CHASM control plane, append-only collection data plane, no stream events in Workflow History, stateless consumers with client-held cursors, long-poll on the CHASM notifier, durable-before-visible, and the same append-then-CAS visibility rule with transaction-chain selection. It reaches those independently, which is worth something on its own. + +Three differences of substance: + +- **It is workflow-attached only.** Standalone streams are an explicit non-goal, on the grounds that a standalone stream turns a Workflow publisher into a cross-execution write needing an outbox. We support both, and reach the same conclusion by a different route: a Workflow publishing to a stream it does not own uses the RPC, not a command, so no cross-execution transaction arises. +- **It does bucket 1 only.** Pushing into a consuming Workflow is a non-goal, so there is no Path C equivalent. +- **It uses a dedicated persistence facet**, like Johann's, rather than reusing `history_node`. + +### The flaw it found in ours + +Its §7.3 requires a bounded physical partition, with `partition_id = offset / partition_message_count`. Checking that against our design: `history_node` in Cassandra is `PRIMARY KEY ((tree_id), branch_id, node_id, txn_id)`, so **the partition key is `tree_id` alone**. Every node of a tree lives in one partition. + +That is safe for Workflow History because history is capped by `HistorySizeLimitError` and `HistoryCountLimitError`. It is not safe for a stream, which the 1-pager explicitly wants unbounded. Reusing `history_node` as-is inherits a partition layout whose safety depends on a cap we are deliberately removing. Large Cassandra partitions degrade compaction and read latency well before anything fails outright. + +We had not noticed this. It is the most useful thing in the document. + +The fix keeps the `history_node` reuse and stays O(1): roll to a new tree every `bucketSize` offsets, with `bucket = offset / bucketSize` and the tree ID derived deterministically from `(streamID, bucket)` via `uuid.NewSHA1`. No index, because the mapping is arithmetic. Reads spanning a boundary issue one range read per bucket, and whole-bucket truncation becomes `DeleteHistoryBranch` rather than per-row tombstones, which is the same benefit Codex gets from buckets. + +Bucketing does not weaken the transaction-chain protection, though the argument is not obvious and needs a test. Bucket boundaries are offset-aligned, so any stale node is either the bucket's first node, where it shares a node ID with the real first node and loses on transaction ID, or it is preceded within its bucket by valid nodes, where the chain rule applies as it does within a single tree. + +### The other gaps it exposed + +- **Continue-as-new for an attached stream.** Ours says continue-as-new needs no handling. That is true for a standalone stream and false for an attached one, where the component lives in a run that is about to be superseded. Codex copies the bounded state into the successor and marks the old child `REDIRECTED` with the new run ID, so an in-flight long poll follows the chain without losing its cursor. We had no answer. +- **Reset.** Ours says the workflow rewinds and the stream does not, which is the right conclusion with none of the mechanism. Codex carries the latest stream state forward into the reset snapshot rather than the state at the reset point, and works through the append-versus-reset race on the current-execution condition. Ours had nothing on the race. +- **Missing data reads as data loss, not as an empty stream** (invariant 3.2.6). Ours would return an empty range if branch data vanished, which is the worst possible failure for a durable stream. We get this more cheaply than Codex does, because our control state already holds the frontier: if a read returns fewer nodes than `HeadOffset` implies, that is `DataLoss`. Codex needs a separate manifest for it because their equivalent state can be rebuilt from history. +- **Different content under a retried sequence must be rejected** (invariant 3.4.3). Ours returns the recorded offsets on a sequence match without comparing content, so a client bug silently drops data. +- **The event-sourcing boundary needs explicit sign-off.** Both designs make stream control non-replayable auxiliary CHASM state. Codex names that as an architecture exception requiring History and CHASM owner approval plus a `docs/architecture` update. Ours does the same thing without flagging it as a governance item. + +### Where we disagree + +Codex defers group commit until measurement proves it necessary; we took it from Johann. Johann's own back-of-envelope says it is load-bearing on Cassandra, and Cassandra is out of prototype scope, so deferring the implementation while keeping the design is the honest position. Moving ours to deferred. + +On the facet, Codex's §7.1 gives five reasons for a dedicated store. Bucketing answers two of them (bounded partitions, truncation on buckets rather than branches). One does not apply to us, since our tree is not tied to a run. We verified the raw paths do not assume History Events. The remaining reason, different replication and garbage-collection ownership metadata, is real and deferred. So the facet case is weaker after bucketing, but not zero, and it is the strongest remaining argument against our storage choice. + +### What we do not take + +The manifest `PREPARED` / `ATTACHED` / `CLOSED` state machine and the quarantined-child sentinel. The invariant behind them is right and we are adopting it; the machinery exists to solve a problem we do not have, because our head authority is not reconstructible from history. Adding a two-phase manifest attachment would put a retryable failure window into the middle of every first append for a diagnostic we can get by comparing a read against the frontier. + +--- + +## 7. What our design has that the others do not + +Stated plainly so it can be attacked: + +1. **No new storage.** Reusing `history_node` removes two tables, four schema migrations, an index structure with its own offload path, and a sweeper. +2. **No commit protocol.** The clip invariant plus the transaction-ID chain gives exactly-once without a prepare phase, which is why no TLA+ spec is required to trust it. +3. **No dependency on CHASM partial reads.** `OSS-4917` and `OSS-4918` are both still `To Do`. Johann's revised plan (analysis §5.1) puts streams on a shared CHASM collection primitive owned by another team, which is the current critical path. Payload bytes never enter the CHASM tree in our design, so that dependency disappears. This is a scheduling argument, not an architectural one, but D1 is due at the September 14 check-in. +4. **In-workflow consumption at no per-item history cost**, and exactly-once without user-side dedup. Option 7 also targets bucket 2, so this is no longer unique; what is different is that ours needs no reset protocol and no idle heuristic, at the price of server work and Temporal-side storage cost. +5. **Replication is inherited** rather than designed, because history-node data already replicates. + +And the honest counterweight: **ours is a design, theirs are working code.** Johann's has a verified spec and a passing end-to-end test. Max's has a conformance suite and around 40 ADRs recording failure modes we have not hit yet. The claims in the table above are unmeasured. The benchmark is what settles it. + +--- + +## 8. Changes we are making + +Applied to both design documents. + +| # | Change | Source | +|---|---|---| +| 1 | Record the consumed range on `WorkflowTaskCompleted` rather than as a separate event, and record it on **every** task for a subscribed stream, including empty ranges. First record for a subscription carries the resolved start offset. | Max ADR-005 (correctness fix) | +| 2 | Strengthen the clip-invariant argument to cite the transaction-ID chain rule, not just same-node-ID resolution. | Johann's orphan objection | +| 3 | Eager orphan trim via `TrimHistoryBranch` as a pure task after a failed append. | Johann's orphan objection | +| 4 | Group commit: coalesce concurrent appends into one CHASM transition. | Johann | +| 5 | Per-producer write fence, distinct from close. | Max ADR-040 | +| 6 | Attached-stream identity keyed on first execution run ID. | Max | +| 7 | Bound Path C delivery by record count as well as bytes. | Max ADR-026, ADR-007 | +| 8 | Missing data on replay blocks with a retryable task failure rather than a nondeterminism error. | Max ADR-014 | +| 9 | Inline retention truncation at end of publish, with consumer pin. No sweeper. | Johann D7 | +| 10 | Multi-topic with subscribe-time filtering; add `ListStreams`. | Johann D5, operators | +| 11 | State the per-stream throughput ceiling and the positioning explicitly. | Johann §1a | +| 12 | State the codec property: the server never sees plaintext. | Johann §3a | +| 13 | Explicit flush control message from the producer, so a consumer does not wait out an idle timeout. | Option 7 | +| 14 | Per-topic sequence number carried alongside the global offset. | Option 7 | +| 15 | **Offset bucketing.** Roll to a new history-node tree every N offsets so a Cassandra partition stays bounded. Fixes a real flaw. | Codex §7.3 | +| 16 | Continue-as-new for an attached stream: copy bounded state to the successor and mark the old child redirected so pollers follow the chain. | Codex §11.1 | +| 17 | Reset carry-forward: the reset run inherits the current head, not the head at the reset point, with the append-versus-reset race resolved on the current-execution condition. | Codex §11.2 | +| 18 | A missing committed node is `DataLoss`, never an empty stream. | Codex invariant 3.2.6 | +| 19 | Reject a retried sequence carrying different content rather than returning the recorded offsets. | Codex invariant 3.4.3 | +| 20 | Treat the event-sourcing boundary extension as a named review gate needing History and CHASM owner sign-off. | Codex §6.4 | +| 21 | Move group commit from the first cut to deferred; keep the design, drop it from the build. | Codex §18.3 | +| 22 | Stream and topic names must never be used as metric tags. | Codex §16.1 | + +## 9. What we are deliberately not taking + +- **A pluggable external backend.** Complementary product option, not this design. Our read API shape would let it sit behind the same client API later. +- **A dedicated `stream_segments` facet.** Section 5 is the argument. +- **The three-step commit protocol and its TLA+ spec.** Not needed once the storage choice removes the cross-facet problem. If review disagrees with section 5, this comes back with it. +- **Push-based in-workflow delivery via signals.** Superseded by the Bellevue no-user-dedup requirement. +- **Marker annotation grammar with runs and segments.** Not needed when the delivery boundary is the workflow task boundary. + +## 10. What would change our mind + +- The persistence argument in §5 has now been checked with running tests (`common/persistence/tests/history_store_stream_log.go`, passing on SQLite and Postgres): blobs round-trip opaquely on a non-run branch, and the transaction-ID chain drops a stale node from a shrinking retry on both the parsing and raw read paths. A negative control confirms the test is not passing vacuously. So the specific objection quoted in §5 does not hold on OSS storage. +- What that does **not** cover is the SaaS layer. `saas-temporal/walker/` overrides `HistoryBranchUtil`, so minting branches whose tree ID is not a run ID still needs a read of that code. If it turns out to be blocked there, the fallback is Johann's dedicated facet, and most of the rest of our design carries over unchanged. +- If orphan volume under real retry rates is worse than the eager trim can keep up with, the sweeper comes back. +- If measured LWT cost per publish does not come out at 1, the whole persistence argument needs rechecking. +- **The strategic question is not ours to settle.** Option 7 moves stream durability outside Temporal. The Native Streams 1-pager currently states the opposite as a principle: "Durable streams. We are not going to sacrifice durability to improve latency or cost", plus an exactly-once write guarantee. Option 7 provides neither, deliberately. That is a legitimate product choice and the CTO can change the principle, but it should be changed explicitly rather than decided by which prototype lands first. If the answer is that Temporal does not need to own the stream, our design is the wrong bet and the sequencing in §4 is the right one. diff --git a/develop/cassmeasure/gen.py b/develop/cassmeasure/gen.py new file mode 100644 index 00000000000..e33711c86de --- /dev/null +++ b/develop/cassmeasure/gen.py @@ -0,0 +1,49 @@ +"""Generate the CQL for the Option B stream-log partition measurement. + +Three shapes, each in its own partition so tablestats can tell them apart: + A 100k rows of 40 bytes an unbatched token stream, one bucket of 100k offsets + B 100 rows of 2 MB the largest batch the current design permits + C 50k rows of 200 bytes a modestly batched stream +""" +import sys + +NS = "11111111-1111-1111-1111-111111111111" + +def rows(bucket, count, payload_bytes, offset_per_row=1): + blob = "0x" + ("ab" * payload_bytes) + out = [] + off = 0 + for _ in range(count): + out.append( + f"INSERT INTO stream_log (shard_id, namespace_id, collection_id, bucket, " + f"start_offset, end_offset, data, data_encoding) VALUES " + f"(1, {NS}, 'c1', {bucket}, {off}, {off + offset_per_row}, {blob}, 'Proto3');" + ) + off += offset_per_row + return out + +print(""" +CREATE KEYSPACE IF NOT EXISTS streammeasure WITH replication = + {'class': 'SimpleStrategy', 'replication_factor': 1}; +USE streammeasure; +DROP TABLE IF EXISTS stream_log; +CREATE TABLE stream_log ( + shard_id int, + namespace_id uuid, + collection_id text, + bucket bigint, + start_offset bigint, + end_offset bigint, + data blob, + data_encoding text, + PRIMARY KEY ((shard_id, namespace_id, collection_id, bucket), start_offset) +) WITH CLUSTERING ORDER BY (start_offset ASC); +""") + +shape = sys.argv[1] +if shape == "A": + print("\n".join(rows(bucket=0, count=100000, payload_bytes=20))) +elif shape == "B": + print("\n".join(rows(bucket=1, count=50, payload_bytes=512 * 1024, offset_per_row=1000))) +elif shape == "C": + print("\n".join(rows(bucket=2, count=50000, payload_bytes=100))) diff --git a/develop/cassmeasure/reads.cql b/develop/cassmeasure/reads.cql new file mode 100644 index 00000000000..afbedc1c908 --- /dev/null +++ b/develop/cassmeasure/reads.cql @@ -0,0 +1,11 @@ +USE streammeasure; +TRACING ON; +-- Find the batch containing an arbitrary offset: reverse slice, one row. +SELECT start_offset, end_offset FROM stream_log + WHERE shard_id=1 AND namespace_id=11111111-1111-1111-1111-111111111111 + AND collection_id='c1' AND bucket=0 AND start_offset <= 73456 + ORDER BY start_offset DESC LIMIT 1; +-- Read forward from there for a page. +SELECT start_offset, end_offset FROM stream_log + WHERE shard_id=1 AND namespace_id=11111111-1111-1111-1111-111111111111 + AND collection_id='c1' AND bucket=0 AND start_offset >= 73456 LIMIT 100; diff --git a/develop/streambench/README.md b/develop/streambench/README.md new file mode 100644 index 00000000000..4db1aa34ff5 --- /dev/null +++ b/develop/streambench/README.md @@ -0,0 +1,36 @@ +# Bucket 2 comparison harness + +Drives the same workload through two designs so their numbers can be compared: + +- **Option 5**, this branch. Payload in a Temporal-owned log, read with + `workflow.subscribe_stream` and `workflow.read_stream`. Needs the + `moedash/sdk-python` checkout on `moe/AI-198-stream-client`. +- **Option 7**, Max's prototype. Payload in Redis, read with + `external_stream`. Needs `mfateev/sdk-python` on `task/python-sdk-streaming`. + +The two live in different checkouts with different `sdk-core` builds, so each +half runs in its own environment. `common.py` and `observed.py` are shared, which +is what makes the halves comparable: same workload, same latency stamping, same +counters. + +## Running + + docker run -d --name bench-redis -p 6399:6379 redis:7-alpine + ./temporal-server --root --config . --env development --allow-no-auth start + +The server config must expose Prometheus, which `development-sqlite.yaml` does +on `127.0.0.1:8000`. Then, from each checkout: + + cd && uv run python develop/streambench/run_option5.py 127.0.0.1:7333 + cd && uv run python develop/streambench/run_option7.py 127.0.0.1:7333 + python3 develop/streambench/compare.py + +## Reading the output + +Temporal work comes from the server's own Prometheus counters and Redis work +from `info commandstats`, so neither design is measured by a mechanism the other +does not use. Count them as counts. Redis is in-memory and SQLite is on disk, so +they are not the same unit of cost. + +Both halves publish one token per append. That is the natural shape for LLM +tokens and the worst case for Option 5, whose guidance is to batch. diff --git a/develop/streambench/common.py b/develop/streambench/common.py new file mode 100644 index 00000000000..2ee263409e8 --- /dev/null +++ b/develop/streambench/common.py @@ -0,0 +1,137 @@ +"""Shared measurement plumbing for the bucket 2 comparison. + +Both halves import this, so the workload, the latency discipline and the +counters are identical by construction. A benchmark whose halves measure +themselves differently measures the harness. +""" + +from __future__ import annotations + +import json +import subprocess +import time +import urllib.request +from dataclasses import dataclass, field, asdict + + +@dataclass(frozen=True) +class Workload: + """What both designs are asked to do. Identical for each.""" + + token_rate: int = 40 + duration_s: int = 20 + message_bytes: int = 20 + + @property + def total_tokens(self) -> int: + return self.token_rate * self.duration_s + + +def scrape_temporal_ops(addr: str = "127.0.0.1:8000") -> dict[str, float]: + """Persistence request counters, by operation. + + Read from the server's own Prometheus endpoint rather than from either SDK, + so neither design can be measured by a mechanism the other does not use. + """ + out: dict[str, float] = {} + try: + with urllib.request.urlopen(f"http://{addr}/metrics", timeout=10) as resp: + body = resp.read().decode() + except Exception: + return out + for line in body.splitlines(): + if not line.startswith("persistence_requests"): + continue + if "{" not in line: + continue + labels, value = line.rsplit(" ", 1) + op = "" + for part in labels[labels.index("{") + 1 : labels.rindex("}")].split(","): + k, _, v = part.partition("=") + if k.strip() == "operation": + op = v.strip().strip('"') + out[op] = out.get(op, 0.0) + float(value) + return out + + +def scrape_redis_ops(container: str = "bench-redis", port: int = 6399) -> dict[str, int]: + """Redis command counts, so the cost that moved out of Temporal is still counted. + + Reads a local server first and falls back to a container, because either is + a legitimate way to run the Option 7 half and a silently empty scrape + reports the cost that moved to Redis as zero. + """ + raw = "" + for cmd in ( + ["redis-cli", "-p", str(port), "info", "commandstats"], + ["docker", "exec", container, "redis-cli", "info", "commandstats"], + ): + try: + done = subprocess.run(cmd, capture_output=True, text=True, timeout=20) + except Exception: + continue + if done.returncode == 0 and "cmdstat_" in done.stdout: + raw = done.stdout + break + if not raw: + raise RuntimeError("no Redis commandstats from either a local server or a container") + out: dict[str, int] = {} + for line in raw.splitlines(): + if not line.startswith("cmdstat_"): + continue + name, _, rest = line.partition(":") + for part in rest.split(","): + k, _, v = part.partition("=") + if k == "calls": + out[name[len("cmdstat_"):]] = int(v) + return out + + +def delta(after: dict, before: dict) -> dict: + keys = set(after) | set(before) + return {k: after.get(k, 0) - before.get(k, 0) for k in keys if after.get(k, 0) - before.get(k, 0)} + + +def percentile(values: list[float], q: float) -> float: + if not values: + return 0.0 + s = sorted(values) + return s[min(len(s) - 1, int(len(s) * q))] + + +@dataclass +class Result: + design: str + workload: dict + tokens_published: int = 0 + tokens_observed: int = 0 + latency_p50_ms: float = 0.0 + latency_p90_ms: float = 0.0 + latency_p99_ms: float = 0.0 + latency_max_ms: float = 0.0 + latency_first10_ms: list = field(default_factory=list) + latency_last10_ms: list = field(default_factory=list) + wall_s: float = 0.0 + temporal_ops: dict = field(default_factory=dict) + redis_ops: dict = field(default_factory=dict) + workflow_tasks: int = 0 + workflow_task_seconds: float = 0.0 + history_events: int = 0 + history_bytes: int = 0 + notes: list[str] = field(default_factory=list) + + @property + def temporal_ops_total(self) -> int: + return int(sum(self.temporal_ops.values())) + + @property + def redis_ops_total(self) -> int: + return int(sum(self.redis_ops.values())) + + def write(self, path: str) -> None: + d = asdict(self) + d["temporal_ops_total"] = self.temporal_ops_total + d["redis_ops_total"] = self.redis_ops_total + with open(path, "w") as f: + json.dump(d, f, indent=2) + print(f"wrote {path}") diff --git a/develop/streambench/compare.py b/develop/streambench/compare.py new file mode 100644 index 00000000000..78b80931cc9 --- /dev/null +++ b/develop/streambench/compare.py @@ -0,0 +1,33 @@ +import json + +a = json.load(open("/tmp/bench/option5.json")) +b = json.load(open("/tmp/bench/option7.json")) +n5, n7 = a["tokens_published"], b["tokens_published"] + +def per(v, n): return f"{v/n:.2f}" if n else "n/a" + +rows = [ + ("tokens published", f"{n5}", f"{n7}"), + ("latency p50 ms", f"{a['latency_p50_ms']:.0f}", f"{b['latency_p50_ms']:.0f}"), + ("latency p90 ms", f"{a['latency_p90_ms']:.0f}", f"{b['latency_p90_ms']:.0f}"), + ("latency p99 ms", f"{a['latency_p99_ms']:.0f}", f"{b['latency_p99_ms']:.0f}"), + ("latency max ms", f"{a['latency_max_ms']:.0f}", f"{b['latency_max_ms']:.0f}"), + ("Temporal ops total", f"{a['temporal_ops_total']}", f"{b['temporal_ops_total']}"), + ("Temporal ops / token", per(a['temporal_ops_total'], n5), per(b['temporal_ops_total'], n7)), + ("Redis ops total", f"{a['redis_ops_total']}", f"{b['redis_ops_total']}"), + ("Redis ops / token", per(a['redis_ops_total'], n5), per(b['redis_ops_total'], n7)), + ("TOTAL io ops / token", per(a['temporal_ops_total']+a['redis_ops_total'], n5), + per(b['temporal_ops_total']+b['redis_ops_total'], n7)), + ("history events", f"{a['history_events']}", f"{b['history_events']}"), + ("history bytes", f"{a['history_bytes']}", f"{b['history_bytes']}"), + ("history bytes / token", per(a['history_bytes'], n5), per(b['history_bytes'], n7)), +] + +w = max(len(r[0]) for r in rows) + 2 +print(f"{'':<{w}}{'Option 5 (Temporal log)':>26}{'Option 7 (Redis)':>20}") +print("-" * (w + 46)) +for name, x, y in rows: + print(f"{name:<{w}}{x:>26}{y:>20}") +print() +print("Option 5 top ops:", sorted(a["temporal_ops"].items(), key=lambda k: -k[1])[:5]) +print("Option 7 top ops:", sorted(b["redis_ops"].items(), key=lambda k: -k[1])[:5]) diff --git a/develop/streambench/observed.py b/develop/streambench/observed.py new file mode 100644 index 00000000000..2134913a67e --- /dev/null +++ b/develop/streambench/observed.py @@ -0,0 +1,43 @@ +"""Latency sink both halves write into from inside the Workflow sandbox. + +A Workflow's return value only exists for the live run, and wall-clock time is +not reachable from Workflow code. This module is passed through the sandbox, so +the stamping happens here where the real clock still is. Both designs use the +same module, so their latency numbers are produced identically. + +This state belongs to the worker process, not to the Workflow, so it does not +rewind when a Workflow task replays. Counting a token by identity is what keeps +a replayed observation from being counted twice. Keying on replay flags would +not: an SDK reports a sticky cache hit as a replay, which would drop live +observations instead. +""" + +from __future__ import annotations + +import time + +LATENCIES_MS: list[float] = [] +COUNT: list[int] = [0] + +_SEEN: set[str] = set() + + +def observe(token: str) -> None: + """Stamp and count one token, at most once however often it is replayed. + + The token carries its own send time, so it is unique per token and doubles + as the identity that makes this idempotent. The first observation is the + live one, since a replay can only follow it, so the latency stays honest. + """ + if token in _SEEN: + return + _SEEN.add(token) + sent_epoch_ms = float(token.split("|", 1)[0]) + LATENCIES_MS.append(time.time() * 1000.0 - sent_epoch_ms) + COUNT[0] += 1 + + +def reset() -> None: + LATENCIES_MS.clear() + COUNT[0] = 0 + _SEEN.clear() diff --git a/develop/streambench/option5.json b/develop/streambench/option5.json new file mode 100644 index 00000000000..fe578002ab8 --- /dev/null +++ b/develop/streambench/option5.json @@ -0,0 +1,72 @@ +{ + "design": "option5-temporal-log", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 267.8310546875, + "latency_p90_ms": 773.27099609375, + "latency_p99_ms": 994.553955078125, + "latency_max_ms": 1029.843994140625, + "latency_first10_ms": [ + 6.2, + 1001.6, + 974.0, + 946.8, + 920.4, + 892.7, + 865.2, + 837.7, + 810.2, + 782.8 + ], + "latency_last10_ms": [ + 197.7, + 169.4, + 138.1, + 108.1, + 76.6, + 44.3, + 66.1, + 32.9, + 127.1, + 99.1 + ], + "wall_s": 24.25420594215393, + "temporal_ops": { + "GetCurrentClusterMetadata": 1.0, + "RangeCompleteReplicationTasks": 1.0, + "RangeCompleteVisibilityTasks": 1.0, + "ListClusterMetadata": 4.0, + "GetOutboundTasks": 1.0, + "ListNamespaces": 48.0, + "ReadHistoryBranch": 69.0, + "GetTransferTasks": 70.0, + "RangeCompleteTransferTasks": 1.0, + "UpdateWorkflowExecution": 1690.0, + "GetTaskQueue": 59.0, + "GetTaskQueueUserData": 1.0, + "UpdateTaskQueue": 9.0, + "ListNexusEndpoints": 2.0, + "RangeCompleteTimerTasks": 1.0, + "GetVisibilityTasks": 1.0, + "GetCurrentExecution": 832.0, + "RangeCompleteOutboundTasks": 1.0, + "GetNamespace": 3156.0, + "GetTimerTasks": 51.0, + "UpsertClusterMembership": 7.0 + }, + "redis_ops": { + "info": 1 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 222, + "history_bytes": 27983, + "notes": [], + "temporal_ops_total": 6006, + "redis_ops_total": 1 +} \ No newline at end of file diff --git a/develop/streambench/option7.json b/develop/streambench/option7.json new file mode 100644 index 00000000000..efef76a6242 --- /dev/null +++ b/develop/streambench/option7.json @@ -0,0 +1,75 @@ +{ + "design": "option7-external-redis", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 2.998046875, + "latency_p90_ms": 4.666748046875, + "latency_p99_ms": 8.6279296875, + "latency_max_ms": 31.7822265625, + "latency_first10_ms": [ + 3.4, + 1.7, + 1.4, + 1.7, + 1.4, + 2.9, + 2.9, + 1.7, + 5.5, + 1.7 + ], + "latency_last10_ms": [ + 4.1, + 3.7, + 4.1, + 4.9, + 11.9, + 4.0, + 4.7, + 3.0, + 3.8, + 4.7 + ], + "wall_s": 22.2182719707489, + "temporal_ops": { + "RangeCompleteVisibilityTasks": 1.0, + "UpdateWorkflowExecution": 4.0, + "GetTaskQueue": 11.0, + "ReadHistoryBranch": 2.0, + "GetCurrentExecution": 2.0, + "RangeCompleteTimerTasks": 1.0, + "GetTaskQueueUserData": 1.0, + "GetTimerTasks": 20.0, + "UpdateShard": 1.0, + "UpdateTaskQueue": 9.0, + "RangeCompleteTransferTasks": 1.0, + "ListNamespaces": 44.0, + "GetVisibilityTasks": 1.0, + "ListNexusEndpoints": 2.0, + "UpsertClusterMembership": 8.0 + }, + "redis_ops": { + "xadd": 800, + "evalsha": 800, + "hset": 800, + "xrange": 541, + "hello": 1, + "info": 1, + "hget": 800, + "hgetall": 4, + "client|setinfo": 2, + "xread": 803 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 23, + "history_bytes": 24140, + "notes": [], + "temporal_ops_total": 108, + "redis_ops_total": 4552 +} \ No newline at end of file diff --git a/develop/streambench/run_option5.py b/develop/streambench/run_option5.py new file mode 100644 index 00000000000..73c22eaafe2 --- /dev/null +++ b/develop/streambench/run_option5.py @@ -0,0 +1,87 @@ +"""Bucket 2 through Option 5: payload in a Temporal-owned log, Workflow reads it. + +Run inside the moedash/sdk-python checkout. Same workload, same latency +discipline and same counters as the Option 7 half. +""" + +from __future__ import annotations + +import asyncio +import sys +import time +import uuid + +sys.path.insert(0, "/tmp/bench") + +import common +import observed + +from temporalio import workflow +from temporalio.client import Client +from temporalio.client_stream import StreamClient +from temporalio.worker import Worker + +from wf5 import ConsumeWorkflow + + +async def main() -> None: + wl = common.Workload() + target = sys.argv[1] if len(sys.argv) > 1 else "127.0.0.1:7333" + result = common.Result(design="option5-temporal-log", workload=vars(wl)) + + client = await Client.connect(target) + stream_client = StreamClient.connect(target, client.namespace) + observed.reset() + + stream_id = f"bench5-{uuid.uuid4().hex[:8]}" + handle_stream = await stream_client.create(stream_id) + + tq = f"bench5-{uuid.uuid4().hex[:8]}" + wf_id = f"bench5-wf-{uuid.uuid4().hex[:8]}" + filler = "x" * max(0, wl.message_bytes - 14) + + async with Worker(client, task_queue=tq, workflows=[ConsumeWorkflow]): + handle = await client.start_workflow( + ConsumeWorkflow.run, [stream_id, wl.total_tokens], id=wf_id, task_queue=tq + ) + await asyncio.sleep(1.0) + + before_t = common.scrape_temporal_ops() + before_r = common.scrape_redis_ops() + started = time.time() + + interval = 1.0 / wl.token_rate + for _ in range(wl.total_tokens): + body = f"{time.time() * 1000.0}|{filler}".encode() + await handle_stream.append(body) + result.tokens_published += 1 + await asyncio.sleep(interval) + + try: + result.tokens_observed = await asyncio.wait_for(handle.result(), timeout=120) + except asyncio.TimeoutError: + result.notes.append("workflow did not finish inside 120s") + result.tokens_observed = observed.COUNT[0] + + result.wall_s = time.time() - started + result.temporal_ops = common.delta(common.scrape_temporal_ops(), before_t) + result.redis_ops = common.delta(common.scrape_redis_ops(), before_r) + + lat = observed.LATENCIES_MS + result.latency_p50_ms = common.percentile(lat, 0.50) + result.latency_p90_ms = common.percentile(lat, 0.90) + result.latency_p99_ms = common.percentile(lat, 0.99) + result.latency_max_ms = max(lat) if lat else 0.0 + # Where the slow ones sit tells a cold start apart from a real tail. + result.latency_first10_ms = [round(x, 1) for x in lat[:10]] + result.latency_last10_ms = [round(x, 1) for x in lat[-10:]] + + desc = await handle.describe() + result.history_events = desc.raw_description.workflow_execution_info.history_length + result.history_bytes = desc.raw_description.workflow_execution_info.history_size_bytes + await stream_client.close() + result.write("/tmp/bench/option5.json") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/develop/streambench/run_option7.py b/develop/streambench/run_option7.py new file mode 100644 index 00000000000..45b78c2ac70 --- /dev/null +++ b/develop/streambench/run_option7.py @@ -0,0 +1,100 @@ +"""Bucket 2 through Option 7: payload in Redis, Workflow reads it directly. + +Run inside Max's sdk-python checkout so `temporalio.contrib.external_workflow_streams` +resolves. The workload, the latency discipline and the counters come from +common.py, which the Option 5 half imports too. +""" + +from __future__ import annotations + +import asyncio +import sys +import time +import uuid +from datetime import timedelta + +sys.path.insert(0, "/tmp/bench") + +import common +import observed + +from temporalio import workflow +from temporalio.client import Client +from temporalio.worker import Worker +from temporalio.contrib.external_workflow_streams._backend import StreamKey +from temporalio.contrib.external_workflow_streams._codec import StreamPayloadCodec +from temporalio.contrib.external_workflow_streams._record import RecordKind, StreamRecord +from temporalio.contrib.external_workflow_streams._redis import RedisStreamBackend +import temporalio.converter + +from wf7 import ConsumeWorkflow + + +async def main() -> None: + wl = common.Workload() + target = sys.argv[1] if len(sys.argv) > 1 else "127.0.0.1:7333" + result = common.Result(design="option7-external-redis", workload=vars(wl)) + + client = await Client.connect(target) + backend = RedisStreamBackend(url="redis://127.0.0.1:6399") + codec = StreamPayloadCodec(temporalio.converter.DataConverter.default, str) + observed.reset() + + tq = f"bench7-{uuid.uuid4().hex[:8]}" + wf_id = f"bench7-wf-{uuid.uuid4().hex[:8]}" + filler = "x" * max(0, wl.message_bytes - 14) + + async with Worker( + client, task_queue=tq, workflows=[ConsumeWorkflow], + external_stream_backend=backend, + ): + handle = await client.start_workflow( + ConsumeWorkflow.run, wl.total_tokens, id=wf_id, task_queue=tq + ) + desc = await handle.describe() + key = StreamKey( + client.namespace, handle.id, + desc.raw_description.workflow_execution_info.first_run_id, "tokens", + ) + await asyncio.sleep(1.0) + + before_t = common.scrape_temporal_ops() + before_r = common.scrape_redis_ops() + started = time.time() + + interval = 1.0 / wl.token_rate + for i in range(wl.total_tokens): + body = f"{time.time() * 1000.0}|{filler}" + await backend.append( + key, StreamRecord(RecordKind.DATA, await codec.encode(body), "bench", i) + ) + result.tokens_published += 1 + await asyncio.sleep(interval) + + try: + result.tokens_observed = await asyncio.wait_for(handle.result(), timeout=120) + except asyncio.TimeoutError: + result.notes.append("workflow did not finish inside 120s") + result.tokens_observed = observed.COUNT[0] + + result.wall_s = time.time() - started + result.temporal_ops = common.delta(common.scrape_temporal_ops(), before_t) + result.redis_ops = common.delta(common.scrape_redis_ops(), before_r) + + lat = observed.LATENCIES_MS + result.latency_p50_ms = common.percentile(lat, 0.50) + result.latency_p90_ms = common.percentile(lat, 0.90) + result.latency_p99_ms = common.percentile(lat, 0.99) + result.latency_max_ms = max(lat) if lat else 0.0 + # Where the slow ones sit tells a cold start apart from a real tail. + result.latency_first10_ms = [round(x, 1) for x in lat[:10]] + result.latency_last10_ms = [round(x, 1) for x in lat[-10:]] + + desc = await handle.describe() + result.history_events = desc.raw_description.workflow_execution_info.history_length + result.history_bytes = desc.raw_description.workflow_execution_info.history_size_bytes + result.write("/tmp/bench/option7.json") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/develop/streambench/runs/option5-r1.json b/develop/streambench/runs/option5-r1.json new file mode 100644 index 00000000000..454e9e9d183 --- /dev/null +++ b/develop/streambench/runs/option5-r1.json @@ -0,0 +1,67 @@ +{ + "design": "option5-temporal-log", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 545.171142578125, + "latency_p90_ms": 925.349365234375, + "latency_p99_ms": 1024.87109375, + "latency_max_ms": 1034.1640625, + "latency_first10_ms": [ + 6.3, + 1002.2, + 974.4, + 946.7, + 917.9, + 889.5, + 861.1, + 831.9, + 799.7, + 768.9 + ], + "latency_last10_ms": [ + 196.6, + 167.2, + 134.2, + 100.2, + 66.9, + 35.5, + 1010.0, + 978.1, + 947.1, + 915.5 + ], + "wall_s": 25.20454692840576, + "temporal_ops": { + "RangeCompleteTimerTasks": 1.0, + "GetTransferTasks": 28.0, + "RangeCompleteTransferTasks": 1.0, + "RangeCompleteVisibilityTasks": 1.0, + "GetTaskQueueUserData": 1.0, + "GetNamespace": 3200.0, + "UpdateWorkflowExecution": 1652.0, + "GetCurrentExecution": 824.0, + "ListNexusEndpoints": 2.0, + "GetTimerTasks": 54.0, + "RangeCompleteOutboundTasks": 1.0, + "UpdateShard": 1.0, + "UpsertClusterMembership": 10.0, + "ReadHistoryBranch": 28.0, + "ListNamespaces": 52.0, + "RangeCompleteArchivalTasks": 1.0 + }, + "redis_ops": { + "info": 1 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 93, + "history_bytes": 11601, + "notes": [], + "temporal_ops_total": 5857, + "redis_ops_total": 1 +} \ No newline at end of file diff --git a/develop/streambench/runs/option5-r2.json b/develop/streambench/runs/option5-r2.json new file mode 100644 index 00000000000..22eff490da2 --- /dev/null +++ b/develop/streambench/runs/option5-r2.json @@ -0,0 +1,70 @@ +{ + "design": "option5-temporal-log", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 161.81787109375, + "latency_p90_ms": 362.906982421875, + "latency_p99_ms": 790.302978515625, + "latency_max_ms": 984.772216796875, + "latency_first10_ms": [ + 5.8, + 984.8, + 958.0, + 930.5, + 903.1, + 875.7, + 847.8, + 820.3, + 790.3, + 759.4 + ], + "latency_last10_ms": [ + 221.0, + 192.6, + 164.9, + 136.6, + 107.9, + 79.4, + 50.9, + 19.3, + 365.2, + 336.6 + ], + "wall_s": 24.4933602809906, + "temporal_ops": { + "GetOutboundTasks": 1.0, + "GetTimerTasks": 48.0, + "UpdateShard": 1.0, + "RangeCompleteTimerTasks": 1.0, + "GetCurrentExecution": 856.0, + "RangeCompleteReplicationTasks": 1.0, + "UpdateTaskQueue": 9.0, + "ListNamespaces": 48.0, + "ReadHistoryBranch": 107.0, + "UpsertClusterMembership": 7.0, + "GetTaskQueue": 7.0, + "ListNexusEndpoints": 2.0, + "RangeCompleteTransferTasks": 1.0, + "GetCurrentClusterMetadata": 1.0, + "GetTaskQueueUserData": 1.0, + "UpdateWorkflowExecution": 1742.0, + "GetTransferTasks": 108.0, + "GetVisibilityTasks": 1.0, + "GetNamespace": 3116.0 + }, + "redis_ops": { + "info": 1 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 345, + "history_bytes": 43698, + "notes": [], + "temporal_ops_total": 6058, + "redis_ops_total": 1 +} \ No newline at end of file diff --git a/develop/streambench/runs/option5-r3.json b/develop/streambench/runs/option5-r3.json new file mode 100644 index 00000000000..fe578002ab8 --- /dev/null +++ b/develop/streambench/runs/option5-r3.json @@ -0,0 +1,72 @@ +{ + "design": "option5-temporal-log", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 267.8310546875, + "latency_p90_ms": 773.27099609375, + "latency_p99_ms": 994.553955078125, + "latency_max_ms": 1029.843994140625, + "latency_first10_ms": [ + 6.2, + 1001.6, + 974.0, + 946.8, + 920.4, + 892.7, + 865.2, + 837.7, + 810.2, + 782.8 + ], + "latency_last10_ms": [ + 197.7, + 169.4, + 138.1, + 108.1, + 76.6, + 44.3, + 66.1, + 32.9, + 127.1, + 99.1 + ], + "wall_s": 24.25420594215393, + "temporal_ops": { + "GetCurrentClusterMetadata": 1.0, + "RangeCompleteReplicationTasks": 1.0, + "RangeCompleteVisibilityTasks": 1.0, + "ListClusterMetadata": 4.0, + "GetOutboundTasks": 1.0, + "ListNamespaces": 48.0, + "ReadHistoryBranch": 69.0, + "GetTransferTasks": 70.0, + "RangeCompleteTransferTasks": 1.0, + "UpdateWorkflowExecution": 1690.0, + "GetTaskQueue": 59.0, + "GetTaskQueueUserData": 1.0, + "UpdateTaskQueue": 9.0, + "ListNexusEndpoints": 2.0, + "RangeCompleteTimerTasks": 1.0, + "GetVisibilityTasks": 1.0, + "GetCurrentExecution": 832.0, + "RangeCompleteOutboundTasks": 1.0, + "GetNamespace": 3156.0, + "GetTimerTasks": 51.0, + "UpsertClusterMembership": 7.0 + }, + "redis_ops": { + "info": 1 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 222, + "history_bytes": 27983, + "notes": [], + "temporal_ops_total": 6006, + "redis_ops_total": 1 +} \ No newline at end of file diff --git a/develop/streambench/runs/option7-r1.json b/develop/streambench/runs/option7-r1.json new file mode 100644 index 00000000000..a06ff9df7d9 --- /dev/null +++ b/develop/streambench/runs/option7-r1.json @@ -0,0 +1,74 @@ +{ + "design": "option7-external-redis", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 2.9287109375, + "latency_p90_ms": 4.4501953125, + "latency_p99_ms": 6.628662109375, + "latency_max_ms": 30.961181640625, + "latency_first10_ms": [ + 3.3, + 1.5, + 1.3, + 1.3, + 2.8, + 3.0, + 3.5, + 3.4, + 4.0, + 4.5 + ], + "latency_last10_ms": [ + 2.8, + 3.6, + 2.7, + 3.7, + 3.1, + 3.9, + 4.3, + 5.2, + 4.4, + 4.2 + ], + "wall_s": 22.041906118392944, + "temporal_ops": { + "ListNexusEndpoints": 2.0, + "GetTaskQueue": 92.0, + "ListNamespaces": 44.0, + "RangeCompleteTimerTasks": 1.0, + "GetTimerTasks": 18.0, + "UpdateWorkflowExecution": 3.0, + "RangeCompleteVisibilityTasks": 1.0, + "RangeCompleteArchivalTasks": 1.0, + "RangeCompleteOutboundTasks": 1.0, + "GetVisibilityTasks": 1.0, + "UpdateTaskQueue": 88.0, + "GetTaskQueueUserData": 1.0, + "ReadHistoryBranch": 2.0, + "ListClusterMetadata": 4.0, + "GetCurrentExecution": 1.0, + "UpsertClusterMembership": 5.0 + }, + "redis_ops": { + "hget": 800, + "info": 1, + "script|load": 1, + "hset": 800, + "xread": 800, + "evalsha": 801, + "xadd": 800, + "hgetall": 1 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 15, + "history_bytes": 32719, + "notes": [], + "temporal_ops_total": 265, + "redis_ops_total": 4004 +} \ No newline at end of file diff --git a/develop/streambench/runs/option7-r2.json b/develop/streambench/runs/option7-r2.json new file mode 100644 index 00000000000..4b13f21352a --- /dev/null +++ b/develop/streambench/runs/option7-r2.json @@ -0,0 +1,71 @@ +{ + "design": "option7-external-redis", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 3.583984375, + "latency_p90_ms": 5807.152099609375, + "latency_p99_ms": 7796.50390625, + "latency_max_ms": 7984.438720703125, + "latency_first10_ms": [ + 2.4, + 1.3, + 1.3, + 1.4, + 1.6, + 1.3, + 1.3, + 2.4, + 1.8, + 4.8 + ], + "latency_last10_ms": [ + 1196.2, + 1169.3, + 1143.0, + 1116.4, + 1089.9, + 1063.2, + 1036.5, + 1012.1, + 985.4, + 958.6 + ], + "wall_s": 22.965797185897827, + "temporal_ops": { + "UpdateTaskQueue": 9.0, + "ListNamespaces": 44.0, + "RangeCompleteTransferTasks": 1.0, + "GetVisibilityTasks": 1.0, + "GetTaskQueueUserData": 1.0, + "GetCurrentExecution": 1.0, + "RangeCompleteVisibilityTasks": 1.0, + "GetTimerTasks": 22.0, + "ListNexusEndpoints": 2.0, + "RangeCompleteOutboundTasks": 1.0, + "GetTaskQueue": 7.0, + "UpdateWorkflowExecution": 3.0, + "ReadHistoryBranch": 2.0, + "UpsertClusterMembership": 7.0 + }, + "redis_ops": { + "xread": 588, + "hget": 800, + "hset": 800, + "evalsha": 800, + "hgetall": 1, + "info": 1, + "xadd": 800 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 18, + "history_bytes": 23350, + "notes": [], + "temporal_ops_total": 102, + "redis_ops_total": 3790 +} \ No newline at end of file diff --git a/develop/streambench/runs/option7-r3.json b/develop/streambench/runs/option7-r3.json new file mode 100644 index 00000000000..efef76a6242 --- /dev/null +++ b/develop/streambench/runs/option7-r3.json @@ -0,0 +1,75 @@ +{ + "design": "option7-external-redis", + "workload": { + "token_rate": 40, + "duration_s": 20, + "message_bytes": 20 + }, + "tokens_published": 800, + "tokens_observed": 800, + "latency_p50_ms": 2.998046875, + "latency_p90_ms": 4.666748046875, + "latency_p99_ms": 8.6279296875, + "latency_max_ms": 31.7822265625, + "latency_first10_ms": [ + 3.4, + 1.7, + 1.4, + 1.7, + 1.4, + 2.9, + 2.9, + 1.7, + 5.5, + 1.7 + ], + "latency_last10_ms": [ + 4.1, + 3.7, + 4.1, + 4.9, + 11.9, + 4.0, + 4.7, + 3.0, + 3.8, + 4.7 + ], + "wall_s": 22.2182719707489, + "temporal_ops": { + "RangeCompleteVisibilityTasks": 1.0, + "UpdateWorkflowExecution": 4.0, + "GetTaskQueue": 11.0, + "ReadHistoryBranch": 2.0, + "GetCurrentExecution": 2.0, + "RangeCompleteTimerTasks": 1.0, + "GetTaskQueueUserData": 1.0, + "GetTimerTasks": 20.0, + "UpdateShard": 1.0, + "UpdateTaskQueue": 9.0, + "RangeCompleteTransferTasks": 1.0, + "ListNamespaces": 44.0, + "GetVisibilityTasks": 1.0, + "ListNexusEndpoints": 2.0, + "UpsertClusterMembership": 8.0 + }, + "redis_ops": { + "xadd": 800, + "evalsha": 800, + "hset": 800, + "xrange": 541, + "hello": 1, + "info": 1, + "hget": 800, + "hgetall": 4, + "client|setinfo": 2, + "xread": 803 + }, + "workflow_tasks": 0, + "workflow_task_seconds": 0.0, + "history_events": 23, + "history_bytes": 24140, + "notes": [], + "temporal_ops_total": 108, + "redis_ops_total": 4552 +} \ No newline at end of file diff --git a/develop/streambench/wf5.py b/develop/streambench/wf5.py new file mode 100644 index 00000000000..cab2d180978 --- /dev/null +++ b/develop/streambench/wf5.py @@ -0,0 +1,26 @@ +"""Option 5 consumer, in its own module so both halves have the same shape. + +Mirrors wf7.py: subscribe, then drain, stamping observation time in the +passed-through sink. +""" + +from __future__ import annotations + +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + import observed as _observed + + +@workflow.defn +class ConsumeWorkflow: + @workflow.run + async def run(self, args: list) -> int: + stream_id, expected = args[0], int(args[1]) + workflow.subscribe_stream(stream_id, start_offset=0) + seen = 0 + while seen < expected: + for body in await workflow.read_stream(stream_id): + _observed.observe(body.decode()) + seen += 1 + return seen diff --git a/develop/streambench/wf7.py b/develop/streambench/wf7.py new file mode 100644 index 00000000000..3fc897a776b --- /dev/null +++ b/develop/streambench/wf7.py @@ -0,0 +1,31 @@ +"""Option 7 consumer, in its own module so the sandbox never re-imports the backend. + +Max's design refuses a backend import from Workflow code on purpose, which is +why the runner and the Workflow cannot share a module. +""" + +from __future__ import annotations + +from datetime import timedelta + +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from temporalio.contrib.external_workflow_streams._api import external_stream + import observed as _observed + + +@workflow.defn +class ConsumeWorkflow: + @workflow.run + async def run(self, expected: int) -> int: + tokens = external_stream.with_options( + idle_timeout=timedelta(seconds=60) + ).topic("tokens", type=str) + seen = 0 + async for token in tokens.subscribe(): + _observed.observe(token) + seen += 1 + if seen >= expected: + break + return seen diff --git a/develop/streamdemo/main.go b/develop/streamdemo/main.go new file mode 100644 index 00000000000..8ab99de852a --- /dev/null +++ b/develop/streamdemo/main.go @@ -0,0 +1,174 @@ +// Command streamdemo shows a native stream end to end: a producer standing in +// for an LLM-calling activity appends tokens, and a browser watches them arrive +// over SSE. The bridge in the middle is what an application's backend would be. +// +// It needs no SDK and no workflow. That is the point of the path it exercises: +// tokens come from an activity, not from workflow code, so nothing here has to +// wait on the workflow-facing API. +// +// go run ./develop/streamdemo -frontend 127.0.0.1:7233 -namespace default +// open http://127.0.0.1:8088 +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/http" + "strconv" + "strings" + "time" + + commonpb "go.temporal.io/api/common/v1" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +var sentence = strings.Fields( + "Durable streaming means the tokens you are reading right now survive a " + + "server restart, because every one of them was committed before it was " + + "shown to you.") + +func main() { + frontend := flag.String("frontend", "127.0.0.1:7233", "frontend gRPC address") + ns := flag.String("namespace", "default", "namespace") + listen := flag.String("listen", "127.0.0.1:8088", "address to serve the demo on") + flag.Parse() + + conn, err := grpc.NewClient(*frontend, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("dial frontend: %v", err) + } + defer func() { _ = conn.Close() }() + client := streampb.NewStreamServiceClient(conn) + + d := &demo{client: client, namespace: *ns} + http.HandleFunc("/", d.page) + http.HandleFunc("/start", d.start) + http.HandleFunc("/events", d.events) + + log.Printf("streamdemo listening on http://%s", *listen) + server := &http.Server{Addr: *listen, ReadHeaderTimeout: 5 * time.Second} + log.Fatal(server.ListenAndServe()) +} + +type demo struct { + client streampb.StreamServiceClient + namespace string +} + +// start creates a stream and produces into it, batching on an interval the way +// an activity consuming a model's token stream would. +func (d *demo) start(w http.ResponseWriter, r *http.Request) { + streamID := r.URL.Query().Get("stream") + if streamID == "" { + http.Error(w, "stream is required", http.StatusBadRequest) + return + } + + created, err := d.client.CreateStream(r.Context(), &streampb.CreateStreamRequest{ + FrontendRequest: &streampb.CreateStreamInput{ + Namespace: d.namespace, StreamId: streamID, + }, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + runID := created.GetFrontendResponse().GetRunId() + + go d.produce(streamID, runID) + w.WriteHeader(http.StatusAccepted) +} + +func (d *demo) produce(streamID, runID string) { + ctx := context.Background() + for i, word := range sentence { + time.Sleep(120 * time.Millisecond) + _, err := d.client.AddMessages(ctx, &streampb.AddMessagesRequest{ + FrontendRequest: &streampb.AddMessagesInput{ + Namespace: d.namespace, StreamId: streamID, RunId: runID, + ProducerId: "demo", Sequence: int64(i + 1), + Messages: []*streampb.StreamMessage{{ + Body: &commonpb.Payload{Data: []byte(word + " ")}, + Kind: streampb.STREAM_MESSAGE_KIND_DATA, + }}, + }, + }) + if err != nil { + log.Printf("append failed: %v", err) + return + } + } + if _, err := d.client.CloseStream(ctx, &streampb.CloseStreamRequest{ + FrontendRequest: &streampb.CloseStreamInput{ + Namespace: d.namespace, StreamId: streamID, + }, + }); err != nil { + log.Printf("close failed: %v", err) + } +} + +// events bridges the stream to the browser. The reader owns its offset, so a +// reconnecting browser resumes exactly where it stopped by passing the offset +// back rather than by the server remembering anything about it. +func (d *demo) events(w http.ResponseWriter, r *http.Request) { + streamID := r.URL.Query().Get("stream") + from, _ := strconv.ParseInt(r.URL.Query().Get("from"), 10, 64) + + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + + for r.Context().Err() == nil { + resp, err := d.client.PollMessages(r.Context(), &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{ + Namespace: d.namespace, StreamId: streamID, + FromOffset: from, WaitNewMessages: true, + }, + }) + if err != nil { + return + } + out := resp.GetFrontendResponse() + for _, m := range out.GetMessages() { + if _, err := fmt.Fprintf(w, "data: %s\n\n", m.GetBody().GetData()); err != nil { + return + } + } + from = out.GetNextOffset() + flusher.Flush() + + if out.GetClosed() && len(out.GetMessages()) == 0 { + _, _ = fmt.Fprint(w, "event: done\ndata: \n\n") + flusher.Flush() + return + } + } +} + +func (d *demo) page(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = fmt.Fprint(w, ` +Temporal native streams + + +

+ +`) +} diff --git a/docs/proposals/README-api-go-stream-changes.md b/docs/proposals/README-api-go-stream-changes.md new file mode 100644 index 00000000000..0a6da14abf1 --- /dev/null +++ b/docs/proposals/README-api-go-stream-changes.md @@ -0,0 +1,56 @@ +# The API changes Paths A and C need + +Stages 5 and 6 of AI-198 cannot be built without changing the public API, because the `Command.attributes` oneof is closed and has no extension point. This records what changes and how the change is built. + +## Two repositories, not one + +This tripped me up, so it is worth stating plainly. + +- **`temporalio/api`** holds the `.proto` files and nothing else. It has no `go.mod` and generates no Go. That is by design, not an omission. +- **`temporalio/api-go`** is the Go module `go.temporal.io/api`. It carries `temporalio/api` as a git submodule at `proto/api` and regenerates from it with `make update-proto`. + +Changing the API means a branch on each: protos in `api`, regenerated output in `api-go` with its submodule pointed at that branch. + +| Repository | Branch | Contents | +|---|---|---| +| `moedash/api` | `moe/AI-198-stream-protos` | 65 lines of proto, nothing else | +| `moedash/api-go` | `moe/AI-198-stream-commands` | Regenerated module, submodule pointed at the above | + +The server pins `moedash/api-go` by pseudo-version through a `replace` in `go.mod`. + +## What the protos add + +| Change | Why | +|---|---| +| `temporal/api/stream/v1/message.proto` with `StreamMessage`, `StreamSlice`, `StreamCursor` | The public shapes. The library's copies under `chasm/lib/stream/proto` are server-internal and no SDK can import them | +| `COMMAND_TYPE_ADD_STREAM_MESSAGES = 19` | Path A: a workflow publishing to its own stream | +| `AddStreamMessagesCommandAttributes` at field 20 of the `Command` oneof | The oneof is closed, so this is the only way | +| `stream_slices` on `PollWorkflowTaskQueueResponse` | Path C: the slice reaches the worker out of band, so payloads never enter History | +| `stream_cursors` on `WorkflowTaskCompletedEventAttributes` | Path C: only the offset range is recorded, on an event that already exists once per task | + +There is **no new event type**, deliberately. Putting the range on `WorkflowTaskCompleted` is what makes recording an empty range free, and an empty range must be recorded on every task where a subscription is active (see `streaming-detailed-design.md` §8.2). + +## Regenerating + +In an `api-go` checkout with submodules, with the `proto/api` submodule on the proto branch: + +``` +make grpc-install mockgen-install # needs pnpm for the nexus plugin +make update-proto +``` + +Two steps were skipped here and their output taken from upstream unchanged, since nothing in this change touches nexus: + +- `nexus-gen` and `system-nexus` need `pnpm`, which was not set up. They produce `workflowservice/v1/workflowservicenexus` and `systemnexus`. + +Running `go-grpc` on its own is not enough. `make clean` removes the mocks and proxy, and only the full `proto` target puts them back, so the module ends up missing `workflowservicemock`, `operatorservicemock`, and the proxy. + +The wider diff across generated files in the `api-go` branch is `protoc-gen-go` version drift from regenerating, not a change in shape. + +## Verifying + +`tests/api_fork_test.go` round-trips all three new shapes through proto marshalling rather than merely compiling against them. A field added without its descriptor compiles fine and silently drops on the wire, which is the failure that guards against. + +## What is not blocked by any of this + +Path B, an off-shard producer with client consumers, needs none of it and is what the benchmark measures. It is also the path LLM token streaming actually takes, since tokens come from an activity rather than from workflow code. diff --git a/go.mod b/go.mod index 7a9d0965bba..6efde59129e 100644 --- a/go.mod +++ b/go.mod @@ -239,3 +239,5 @@ require ( ) tool golang.org/x/perf/cmd/benchstat + +replace go.temporal.io/api => github.com/moedash/api-go v1.63.6-0.20260828011238-4906d7ab0aa9 diff --git a/go.sum b/go.sum index 95797622c25..3156b71605d 100644 --- a/go.sum +++ b/go.sum @@ -321,6 +321,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moedash/api-go v1.63.6-0.20260828011238-4906d7ab0aa9 h1:6332lzDU4i1afFNbTQZMDsuvzT6bCkRHX1RK77wcBmE= +github.com/moedash/api-go v1.63.6-0.20260828011238-4906d7ab0aa9/go.mod h1:SrlW2JMwVlDP4nRWSNznUFqnSHd+YeMDS1BkYo63HCQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= @@ -479,8 +481,6 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.63.5 h1:c11+kPYHkXXL3UiShPdbMD+xtvqGsbTibUA9ypmiCa4= -go.temporal.io/api v1.63.5/go.mod h1:SrlW2JMwVlDP4nRWSNznUFqnSHd+YeMDS1BkYo63HCQ= go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab h1:99wXW0317BBi49d6xgMdA0EZtvA+xbBUWV4HsTEGEcg= go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab/go.mod h1:hhHijO9XRPIkAflLJJHix61M9FzbRPqk8fSydkcLkqw= go.temporal.io/sdk v1.44.0 h1:suitPDukX74rW3/N1FqvEbZTZVJJsxMKhv0KMa/j7pU= diff --git a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto index de813adcf43..811b0dcc07c 100644 --- a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto +++ b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto @@ -14,6 +14,7 @@ import "temporal/api/history/v1/message.proto"; import "temporal/api/nexus/v1/message.proto"; import "temporal/api/protocol/v1/message.proto"; import "temporal/api/query/v1/message.proto"; +import "temporal/api/stream/v1/message.proto"; import "temporal/api/taskqueue/v1/message.proto"; import "temporal/api/workflow/v1/message.proto"; import "temporal/api/workflowservice/v1/request_response.proto"; @@ -301,6 +302,10 @@ message RecordWorkflowTaskStartedResponse { // as raw_history_bytes (field 21) will be the only field used. temporal.api.history.v1.History raw_history = 20 [deprecated = true]; repeated bytes raw_history_bytes = 21; + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + repeated temporal.api.stream.v1.StreamSlice stream_slices = 22; } // RecordWorkflowTaskStartedResponseWithRawHistory is wire-compatible with RecordWorkflowTaskStartedResponse. @@ -345,6 +350,10 @@ message RecordWorkflowTaskStartedResponseWithRawHistory { // instead of a proto-decoded History. This avoids matching service having to decode history. repeated bytes raw_history = 20 [deprecated = true]; repeated bytes raw_history_bytes = 21; + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + repeated temporal.api.stream.v1.StreamSlice stream_slices = 22; } message RecordActivityTaskStartedRequest { diff --git a/proto/internal/temporal/server/api/matchingservice/v1/request_response.proto b/proto/internal/temporal/server/api/matchingservice/v1/request_response.proto index 33bae6d5c71..afe058cef5b 100644 --- a/proto/internal/temporal/server/api/matchingservice/v1/request_response.proto +++ b/proto/internal/temporal/server/api/matchingservice/v1/request_response.proto @@ -12,6 +12,7 @@ import "temporal/api/history/v1/message.proto"; import "temporal/api/nexus/v1/message.proto"; import "temporal/api/protocol/v1/message.proto"; import "temporal/api/query/v1/message.proto"; +import "temporal/api/stream/v1/message.proto"; import "temporal/api/taskqueue/v1/message.proto"; import "temporal/api/worker/v1/message.proto"; import "temporal/api/workflowservice/v1/request_response.proto"; @@ -62,6 +63,10 @@ message PollWorkflowTaskQueueResponse { // Raw history bytes sent from matching service when history.sendRawHistoryBetweenInternalServices is enabled. // Matching client will deserialize this to History when it receives the response. temporal.api.history.v1.History raw_history = 22; + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + repeated temporal.api.stream.v1.StreamSlice stream_slices = 23; } // PollWorkflowTaskQueueResponseWithRawHistory is wire-compatible with PollWorkflowTaskQueueResponse. @@ -112,6 +117,10 @@ message PollWorkflowTaskQueueResponseWithRawHistory { // When matching client deserializes this to PollWorkflowTaskQueueResponse, this field // will be automatically deserialized to the raw_history field as History. repeated bytes raw_history = 22; + // Slices of the streams this Workflow is consuming, each covering the offset + // range the task may deliver. Payloads ride here and never enter History; + // only the consumed range is recorded, on WorkflowTaskCompleted. + repeated temporal.api.stream.v1.StreamSlice stream_slices = 23; } message PollActivityTaskQueueRequest { diff --git a/schema/cassandra/temporal/schema.cql b/schema/cassandra/temporal/schema.cql index 9d664fe5981..eada7b95184 100644 --- a/schema/cassandra/temporal/schema.cql +++ b/schema/cassandra/temporal/schema.cql @@ -247,3 +247,18 @@ CREATE TABLE nexus_endpoints ) WITH COMPACTION = { 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy' }; + +CREATE TABLE stream_log ( + shard_id int, + namespace_id uuid, + collection_id text, + bucket bigint, -- one bucket is one partition, bounded by offsets and by bytes + start_offset bigint, -- first offset in this batch, and the key a retry addresses + next_offset bigint, -- exclusive, so a reader knows the span without decoding + data blob, + data_encoding text, + PRIMARY KEY ((shard_id, namespace_id, collection_id, bucket), start_offset) +) WITH CLUSTERING ORDER BY (start_offset ASC) + AND COMPACTION = { + 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy' +}; diff --git a/schema/cassandra/temporal/versioned/v1.10/add_stream_log.cql b/schema/cassandra/temporal/versioned/v1.10/add_stream_log.cql new file mode 100644 index 00000000000..9b9f0ad0c6c --- /dev/null +++ b/schema/cassandra/temporal/versioned/v1.10/add_stream_log.cql @@ -0,0 +1,14 @@ +CREATE TABLE stream_log ( + shard_id int, + namespace_id uuid, + collection_id text, + bucket bigint, -- one bucket is one partition, bounded by offsets and by bytes + start_offset bigint, -- first offset in this batch, and the key a retry addresses + next_offset bigint, -- exclusive, so a reader knows the span without decoding + data blob, + data_encoding text, + PRIMARY KEY ((shard_id, namespace_id, collection_id, bucket), start_offset) +) WITH CLUSTERING ORDER BY (start_offset ASC) + AND COMPACTION = { + 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy' +}; diff --git a/schema/cassandra/temporal/versioned/v1.10/manifest.json b/schema/cassandra/temporal/versioned/v1.10/manifest.json index bab20aee84f..5afd4458e17 100644 --- a/schema/cassandra/temporal/versioned/v1.10/manifest.json +++ b/schema/cassandra/temporal/versioned/v1.10/manifest.json @@ -1,6 +1,8 @@ { "CurrVersion": "1.10", "MinCompatibleVersion": "1.0", - "Description": "create nexus_incoming_services table", - "SchemaUpdateCqlFiles": ["nexus_incoming_services.cql"] + "Description": "Add stream_log table for server-side streams", + "SchemaUpdateCqlFiles": [ + "add_stream_log.cql" + ] } diff --git a/schema/sqlite/v3/temporal/schema.sql b/schema/sqlite/v3/temporal/schema.sql index d7056640a14..f759d240cd5 100644 --- a/schema/sqlite/v3/temporal/schema.sql +++ b/schema/sqlite/v3/temporal/schema.sql @@ -319,6 +319,18 @@ CREATE TABLE history_node ( data_encoding VARCHAR(16) NOT NULL, PRIMARY KEY (shard_id, tree_id, branch_id, node_id, txn_id) ); +CREATE TABLE stream_log ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + collection_id VARCHAR(255) NOT NULL, + bucket BIGINT NOT NULL, + start_offset BIGINT NOT NULL, + -- + next_offset BIGINT NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, namespace_id, collection_id, bucket, start_offset) +); -- history eventsV2: history_tree stores branch metadata CREATE TABLE history_tree ( diff --git a/schema/sqlite/v3/temporal/versioned/v0.10/add_stream_log.sql b/schema/sqlite/v3/temporal/versioned/v0.10/add_stream_log.sql new file mode 100644 index 00000000000..6c523d75ba2 --- /dev/null +++ b/schema/sqlite/v3/temporal/versioned/v0.10/add_stream_log.sql @@ -0,0 +1,12 @@ +CREATE TABLE stream_log ( + shard_id INT NOT NULL, + namespace_id BINARY(16) NOT NULL, + collection_id VARCHAR(255) NOT NULL, + bucket BIGINT NOT NULL, + start_offset BIGINT NOT NULL, + -- + next_offset BIGINT NOT NULL, + data MEDIUMBLOB NOT NULL, + data_encoding VARCHAR(16) NOT NULL, + PRIMARY KEY (shard_id, namespace_id, collection_id, bucket, start_offset) +); diff --git a/schema/sqlite/v3/temporal/versioned/v0.10/manifest.json b/schema/sqlite/v3/temporal/versioned/v0.10/manifest.json index 7bb9a2e7313..aa3cfecdb23 100644 --- a/schema/sqlite/v3/temporal/versioned/v0.10/manifest.json +++ b/schema/sqlite/v3/temporal/versioned/v0.10/manifest.json @@ -1,8 +1,8 @@ { "CurrVersion": "0.10", "MinCompatibleVersion": "0.1", - "Description": "Adds tasks_v2 table for fairness tasks", + "Description": "Add stream_log table for server-side streams", "SchemaUpdateCqlFiles": [ - "tasks_v2.sql" + "add_stream_log.sql" ] } diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 651551de210..1546d14ad8d 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -14,6 +14,7 @@ import ( nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" chasmscheduler "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + chasmstream "go.temporal.io/server/chasm/lib/stream/service" chasmtests "go.temporal.io/server/chasm/lib/tests" chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/client" @@ -146,6 +147,7 @@ var Module = fx.Options( chasmworkflow.Module, chasmcallback.Module, activity.FrontendModule, + chasmstream.FrontendModule, fx.Provide(visibility.ChasmVisibilityManagerProvider), fx.Provide(chasm.ChasmVisibilityInterceptorProvider), ) @@ -158,6 +160,7 @@ func NewServiceProvider( handler Handler, adminHandler *AdminHandler, operatorHandler *OperatorHandlerImpl, + streamHandler *chasmstream.FrontendHandler, versionChecker *VersionChecker, visibilityMgr manager.VisibilityManager, logger log.SnTaggedLogger, @@ -173,6 +176,7 @@ func NewServiceProvider( handler, adminHandler, operatorHandler, + streamHandler, versionChecker, visibilityMgr, logger, diff --git a/service/frontend/service.go b/service/frontend/service.go index f48d86c33bc..62da8849873 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -13,6 +13,8 @@ import ( "go.temporal.io/server/chasm/lib/activity" chasmcallback "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + chasmstream "go.temporal.io/server/chasm/lib/stream/service" "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" @@ -458,6 +460,7 @@ type Service struct { handler Handler adminHandler *AdminHandler operatorHandler *OperatorHandlerImpl + streamHandler *chasmstream.FrontendHandler versionChecker *VersionChecker visibilityManager manager.VisibilityManager server *grpc.Server @@ -477,6 +480,7 @@ func NewService( handler Handler, adminHandler *AdminHandler, operatorHandler *OperatorHandlerImpl, + streamHandler *chasmstream.FrontendHandler, versionChecker *VersionChecker, visibilityMgr manager.VisibilityManager, logger log.Logger, @@ -492,6 +496,7 @@ func NewService( handler: handler, adminHandler: adminHandler, operatorHandler: operatorHandler, + streamHandler: streamHandler, versionChecker: versionChecker, visibilityManager: visibilityMgr, logger: logger, @@ -509,6 +514,7 @@ func (s *Service) Start() { workflowservice.RegisterWorkflowServiceServer(s.server, s.handler) adminservice.RegisterAdminServiceServer(s.server, s.adminHandler) operatorservice.RegisterOperatorServiceServer(s.server, s.operatorHandler) + streampb.RegisterStreamServiceServer(s.server, s.streamHandler) reflection.Register(s.server) diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 2a1230a3c04..16c4c7d081a 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -1197,6 +1197,7 @@ func (wh *WorkflowHandler) PollWorkflowTaskQueue(ctx context.Context, request *w StartedTime: matchingResp.StartedTime, Queries: matchingResp.Queries, Messages: matchingResp.Messages, + StreamSlices: matchingResp.StreamSlices, PollerScalingDecision: matchingResp.PollerScalingDecision, }, nil } diff --git a/service/history/api/command_attr_validator.go b/service/history/api/command_attr_validator.go index 8c48b0f99f1..5118a1d4ea8 100644 --- a/service/history/api/command_attr_validator.go +++ b/service/history/api/command_attr_validator.go @@ -661,7 +661,13 @@ func (v *CommandAttrValidator) ValidateCommandSequence( enumspb.COMMAND_TYPE_MODIFY_WORKFLOW_PROPERTIES, enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - enumspb.COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION: + enumspb.COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION, + // Publishing to a stream the workflow owns. Not a close command: + // it appends and returns, scheduling nothing further. + enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + // Subscribing to a stream. Also not closing: it records a cursor + // and the workflow carries on. + enumspb.COMMAND_TYPE_SUBSCRIBE_STREAM: // noop case enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, diff --git a/service/history/api/recordworkflowtaskstarted/api.go b/service/history/api/recordworkflowtaskstarted/api.go index e88a3c6c9c3..98650b444ab 100644 --- a/service/history/api/recordworkflowtaskstarted/api.go +++ b/service/history/api/recordworkflowtaskstarted/api.go @@ -51,6 +51,7 @@ func Invoke( var workflowKey definition.WorkflowKey var resp *historyservice.RecordWorkflowTaskStartedResponseWithRawHistory + var streamAddresses map[string]streamOrigin err = api.GetAndUpdateWorkflowWithNew( ctx, @@ -102,6 +103,12 @@ func Invoke( if err != nil { return nil, err } + // Redelivers whatever range is already staged, so a + // duplicate of the same request hands back the same slice. + resp.StreamSlices, streamAddresses, err = deliverStreamSlices(ctx, shardContext, mutableState) + if err != nil { + return nil, err + } updateAction.Noop = true return updateAction, nil } @@ -238,6 +245,11 @@ func Invoke( return nil, err } + resp.StreamSlices, streamAddresses, err = deliverStreamSlices(ctx, shardContext, mutableState) + if err != nil { + return nil, err + } + return updateAction, nil }, nil, @@ -264,6 +276,12 @@ func Invoke( if err != nil { return nil, err } + + // After the history is attached, because the ranges to re-supply are read + // out of the events being sent. + if err := attachReplaySlices(ctx, workflowKey, workflowKey.GetNamespaceID(), streamAddresses, resp); err != nil { + return nil, err + } return resp, nil } @@ -400,6 +418,7 @@ func CreateRecordWorkflowTaskStartedResponse( Queries: rawResp.Queries, Clock: rawResp.Clock, Messages: rawResp.Messages, + StreamSlices: rawResp.StreamSlices, Version: rawResp.Version, NextPageToken: rawResp.NextPageToken, }, nil diff --git a/service/history/api/recordworkflowtaskstarted/stream_slices.go b/service/history/api/recordworkflowtaskstarted/stream_slices.go new file mode 100644 index 00000000000..435c43b32f0 --- /dev/null +++ b/service/history/api/recordworkflowtaskstarted/stream_slices.go @@ -0,0 +1,506 @@ +package recordworkflowtaskstarted + +import ( + "context" + "slices" + + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + "go.temporal.io/api/serviceerror" + streampb "go.temporal.io/api/stream/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/persistence/serialization" + historyi "go.temporal.io/server/service/history/interfaces" + "google.golang.org/protobuf/proto" +) + +// deliverStreamSlices hands the next range of every stream this workflow +// consumes to the task being started, and stages that range on the cursor so +// the event closing the task can record what was delivered. +// +// The log read runs with the workflow lock held. That is the price of deciding +// a range and staging it in one transaction: staged first and read after, a +// failed read would leave a range that the worker never received but that the +// completion would still record as consumed. +// streamOrigin says where a subscribed stream lives, which decides how its +// payload is read. The payload is component state now, so an external stream is +// read from its own execution and an owned one from the consumer's. +type streamOrigin struct { + external bool + // The name the consumer knows the stream by, which is how an owned one is + // found on the consumer's own component. + name string + + // Where the subscription began, and how far its completed tasks have + // committed. Together they say exactly which offsets replay owes the + // worker, so a re-supply that comes up short can be detected instead of + // silently handing the workflow less than it originally saw. + start int64 + committed int64 +} + +// deliveryFrontier is the offset a delivery clips to. For a stream this +// execution owns it is read directly; for one in another execution it is +// whatever that stream last pushed here, because reading the real value would +// mean reaching across executions while this transaction is open. +func deliveryFrontier( + chasmCtx chasm.Context, + wf *chasmworkflow.Workflow, + name string, + cursor *stream.Cursor, +) (int64, error) { + if cursor.IsExternal() { + return cursor.KnownHead(), nil + } + field, ok := wf.Streams[name] + if !ok { + return 0, serviceerror.NewFailedPreconditionf( + "workflow consumes stream %q, which it neither owns nor subscribed to externally", name) + } + state, err := field.Get(chasmCtx).Snapshot(chasmCtx, struct{}{}) + if err != nil { + return 0, err + } + + // A consumer that fell behind a truncating stream is told so rather than + // handed the rest and left with a hole it cannot see. Nothing holds the + // floor for a consumer any more, on purpose: a floor that waited for the + // slowest reader was never released and turned every cap into a no-op. + if cursor.Offset() < state.GetBaseOffset() { + return 0, serviceerror.NewFailedPreconditionf( + "stream %q was truncated past this consumer: it is at offset %d and the stream now starts at %d", + name, cursor.Offset(), state.GetBaseOffset()) + } + return state.GetHeadOffset(), nil +} + +func readDeliverable( + ctx context.Context, + chasmCtx chasm.Context, + wf *chasmworkflow.Workflow, + namespaceID string, + name string, + cursor *stream.Cursor, + from, to int64, +) ([]*streampb.StreamMessage, int64, error) { + if to <= from { + return nil, from, nil + } + w, err := readWindowFor(ctx, chasmCtx, wf, namespaceID, name, cursor.IsExternal(), + cursor.StreamID(), from, to) + if err != nil { + return nil, 0, err + } + // The collected run is contiguous from `from`, so the byte cap recomputes + // the same end offset the read would have reported. + collected, _, err := stream.CollectMessages( + w.Blobs, w.Starts, from, w.To, stream.MaxConsumeItemsPerTask, nil) + if err != nil { + return nil, 0, err + } + // Cap before converting: the byte budget applies to the run as stored, and + // trimming decides how far the recorded range reaches. + collected, readTo := stream.CapByBytes(collected, from, stream.MaxConsumeBytesPerTask) + return stream.ToAPIMessages(collected), readTo, nil +} + +// readWindowFor reads a range from whichever component holds it. +// +// An external stream resolves through the local shard controller, so a stream +// on a shard this host does not own is not reachable. An owned stream is read +// from the consumer's own already-loaded component, which is both cheaper and +// the only safe order: re-entering this execution through the engine while its +// task is being built would contend with the lock already held. +func readWindowFor( + ctx context.Context, + chasmCtx chasm.Context, + wf *chasmworkflow.Workflow, + namespaceID string, + name string, + external bool, + streamID string, + from, to int64, +) (stream.Window, error) { + req := stream.WindowRequest{From: from, MaxMessages: int32(to - from)} + if !external { + s := wf.OwnedStream(chasmCtx, name) + if s == nil { + return stream.Window{To: from}, nil + } + return s.ReadWindow(chasmCtx, req) + } + return chasm.ReadComponent(ctx, + chasm.NewComponentRef[*stream.Stream](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: streamID, + }), + (*stream.Stream).ReadWindow, req) +} + +// DeliverStreamSlices hands the next range to a task built outside this +// package. The inline task returned by RespondWorkflowTaskCompleted is built by +// its own handler, so without this a subscribed workflow gets no data on the +// dispatch path every current SDK asks for. +// +// Only the live range. That task is always sticky, so the worker still holds +// the execution and has no ranges to replay. +func DeliverStreamSlices( + ctx context.Context, + shardContext historyi.ShardContext, + ms historyi.MutableState, +) ([]*streampb.StreamSlice, error) { + live, _, err := deliverStreamSlices(ctx, shardContext, ms) + return live, err +} + +func deliverStreamSlices( + ctx context.Context, + shardContext historyi.ShardContext, + ms historyi.MutableState, +) ([]*streampb.StreamSlice, map[string]streamOrigin, error) { + if !ms.HasChasmWorkflowComponent() { + return nil, nil, nil + } + // Read-only first. Reaching the component mutably marks it dirty, and a + // workflow with no subscription should not pay a node in its transaction + // for every task it runs. + if readOnly, _, err := ms.ChasmWorkflowComponentReadOnly(ctx); err != nil { + return nil, nil, err + } else if len(readOnly.StreamCursors) == 0 { + return nil, nil, nil + } + + wf, chasmCtx, err := ms.ChasmWorkflowComponent(ctx) + if err != nil { + return nil, nil, err + } + + names := make([]string, 0, len(wf.StreamCursors)) + for name := range wf.StreamCursors { + names = append(names, name) + } + // Delivery order has to be stable, because the completion records these + // ranges in the order they were produced. + slices.Sort(names) + + maxItems := stream.MaxConsumeItemsPerTask + namespaceID := ms.GetExecutionInfo().GetNamespaceId() + + slicesOut := make([]*streampb.StreamSlice, 0, len(names)) + addresses := make(map[string]streamOrigin, len(names)) + for _, name := range names { + cursor := wf.StreamCursors[name].Get(chasmCtx) + + head, err := deliveryFrontier(chasmCtx, wf, name, cursor) + if err != nil { + return nil, nil, err + } + + // A range already staged is redelivered unchanged. The same task can be + // started more than once, and letting the second attempt pick up newer + // data would hand the workflow a different range than the one the + // completion is going to record. + from, to, restaged := cursor.Pending() + if !restaged { + from = cursor.Offset() + // Clip to the frontier. Bytes reach the log before the transaction + // that makes them visible commits, so reading past head risks + // delivering an offset whose content a retry could still replace. + to = min(from+int64(maxItems), head) + } + + messages, next, err := readDeliverable( + ctx, chasmCtx, wf, namespaceID, name, cursor, from, to) + if err != nil { + return nil, nil, err + } + + if restaged { + // The staged range is the one the completion will record, so the + // worker has to be handed exactly that. A re-read that comes back + // short would otherwise deliver less than history claims was + // consumed, and replay would then disagree with the original run. + if next != to { + return nil, nil, serviceerror.NewInternalf( + "stream %q staged range [%d,%d) re-read as [%d,%d)", + cursor.StreamID(), from, to, from, next) + } + } else if err := cursor.StagePending(chasmCtx, from, next); err != nil { + return nil, nil, err + } + + // Attached even when empty. A task that saw nothing still has to record + // that it saw nothing, and the slice is what the completion reads. + slicesOut = append(slicesOut, &streampb.StreamSlice{ + StreamId: cursor.StreamID(), + FromOffset: from, + ToOffset: next, + Messages: messages, + }) + addresses[cursor.StreamID()] = streamOrigin{ + external: cursor.IsExternal(), + name: name, + start: cursor.StartOffset(), + committed: cursor.Offset(), + } + } + return slicesOut, addresses, nil +} + +// Bounds on one cold replay's re-supply. History holds offsets and not payloads, +// so replaying a consumer means re-reading every range its completed tasks +// recorded, and that grows with the workflow's whole life rather than with the +// task being started. These cap one response; a consumer past them cannot be +// replayed by this path at all, which is a limit of the design rather than of +// the numbers. +const ( + maxReplayMessages = 100_000 + maxReplayBytes = 64 << 20 +) + +// ownedRange names a range of a stream the consumer owns. +type ownedRange struct { + name string + req stream.WindowRequest +} + +// readRecordedRange re-supplies a range a completed task recorded. +// +// It runs after the execution lock is released, so the consumer's own component +// is read back through the engine like any other. Both kinds resolve through +// the local shard controller, so a stream on a shard this host does not own is +// not reachable. +func readRecordedRange( + ctx context.Context, + consumer definition.WorkflowKey, + origin streamOrigin, + streamID string, + from, to int64, +) (stream.Window, error) { + req := stream.WindowRequest{From: from, MaxMessages: int32(to - from)} + if origin.external { + return chasm.ReadComponent(ctx, + chasm.NewComponentRef[*stream.Stream](chasm.ExecutionKey{ + NamespaceID: consumer.NamespaceID, + BusinessID: streamID, + }), + (*stream.Stream).ReadWindow, req) + } + return chasm.ReadComponent(ctx, + chasm.NewComponentRef[*chasmworkflow.Workflow](chasm.ExecutionKey{ + NamespaceID: consumer.NamespaceID, + BusinessID: consumer.WorkflowID, + }), + func(wf *chasmworkflow.Workflow, cctx chasm.Context, r ownedRange) (stream.Window, error) { + s := wf.OwnedStream(cctx, r.name) + if s == nil { + return stream.Window{To: r.req.From}, nil + } + return s.ReadWindow(cctx, r.req) + }, + ownedRange{name: origin.name, req: req}) +} + +// attachReplaySlices re-supplies the payloads for ranges that earlier workflow +// tasks recorded, keyed by the event that recorded each one. +// +// History holds offsets and never payloads, which is the property the whole +// design rests on. The cost lands here: a worker replaying from History has to +// be handed the same bytes those tasks were given, and the only place they +// exist is the stream's log. The response field alone cannot carry this, +// because it is built once per delivery while a cache miss replays every prior +// task, so each range travels with the id of the event that recorded it. +func attachReplaySlices( + ctx context.Context, + consumer definition.WorkflowKey, + namespaceID string, + addresses map[string]streamOrigin, + resp *historyservice.RecordWorkflowTaskStartedResponseWithRawHistory, +) error { + // Only a workflow with a live subscription has anything to re-supply, and + // that is exactly when delivery attached a slice for the current task. So + // this costs nothing for every other workflow. + if len(addresses) == 0 { + return nil + } + + // A sticky task means the worker still holds the execution, so it has + // nothing to replay. Its history begins at the previous task's completion, + // and that event carries the range that task already consumed, so + // re-supplying it here would hand the workflow the same messages twice. + if resp.GetStickyExecutionEnabled() { + return nil + } + + events, err := eventsOfResponse(resp) + if err != nil { + return err + } + + budget := replayBudget{} + // How far the recorded ranges reach, per stream, so the coverage check + // below can tell a complete re-supply from a short one. + reached := make(map[string]int64, len(addresses)) + + for _, event := range events { + for _, recorded := range event.GetWorkflowTaskCompletedEventAttributes().GetStreamCursors() { + address, ok := addresses[recorded.GetStreamId()] + if !ok { + // A subscription the workflow has since dropped. The range is + // still part of its history, but nothing is consuming it now. + continue + } + + messages, err := replayMessagesFor(ctx, consumer, address, recorded, &budget) + if err != nil { + return err + } + + if to := recorded.GetToOffset(); to > reached[recorded.GetStreamId()] { + reached[recorded.GetStreamId()] = to + } + + // Attached even when empty: the task observed nothing, and replay + // has to reproduce that rather than infer it from an absence. + resp.StreamSlices = append(resp.StreamSlices, &streampb.StreamSlice{ + StreamId: recorded.GetStreamId(), + FromOffset: recorded.GetFromOffset(), + ToOffset: recorded.GetToOffset(), + Messages: messages, + WorkflowTaskCompletedEventId: event.GetEventId(), + }) + } + } + + return checkReplayCoverage(consumer, addresses, reached) +} + +// replayBudget accumulates what one response has already committed to +// re-supplying, across every stream and every recorded range in it. +type replayBudget struct { + messages int + bytes int +} + +// replayMessagesFor re-reads one recorded range, or returns nothing for a range +// that recorded an empty observation. +func replayMessagesFor( + ctx context.Context, + consumer definition.WorkflowKey, + address streamOrigin, + recorded *streampb.StreamCursor, + budget *replayBudget, +) ([]*streampb.StreamMessage, error) { + if recorded.GetToOffset() <= recorded.GetFromOffset() { + return nil, nil + } + + w, err := readRecordedRange(ctx, consumer, address, + recorded.GetStreamId(), recorded.GetFromOffset(), recorded.GetToOffset()) + if err != nil { + return nil, replayReadError(consumer, recorded, err) + } + collected, _, err := stream.CollectMessages( + w.Blobs, w.Starts, + recorded.GetFromOffset(), recorded.GetToOffset(), + int(recorded.GetToOffset()-recorded.GetFromOffset()), nil) + if err != nil { + return nil, err + } + messages := stream.ToAPIMessages(collected) + + budget.messages += len(messages) + for _, m := range messages { + budget.bytes += proto.Size(m) + } + // Bounded because every prior task's range is re-read into one response, so + // a long-lived consumer's cold replay grows with its whole history. Refused + // rather than trimmed: a short re-supply is what replay cannot survive. + if budget.messages > maxReplayMessages || budget.bytes > maxReplayBytes { + return nil, serviceerror.NewFailedPreconditionf( + "replaying workflow %q needs more than %d messages or %d bytes of stream history to re-supply; "+ + "the consumed ranges cannot be re-delivered in one response", + consumer.GetWorkflowID(), maxReplayMessages, maxReplayBytes) + } + return messages, nil +} + +// checkReplayCoverage refuses a re-supply that stops short of what History says +// the consumer consumed. +// +// The events carried on the response are one page. A consumer whose recording +// events run past it would be handed only part of what it originally saw, and +// would then replay against fewer messages than the first run had. The cursor +// knows how far it has committed, so that is checked rather than assumed. +func checkReplayCoverage( + consumer definition.WorkflowKey, + addresses map[string]streamOrigin, + reached map[string]int64, +) error { + for streamID, address := range addresses { + got, ok := reached[streamID] + if !ok { + got = address.start + } + if got < address.committed { + return serviceerror.NewFailedPreconditionf( + "workflow %q consumed stream %q through offset %d but its history page only records through %d; "+ + "re-supplying the rest needs the events beyond this page", + consumer.GetWorkflowID(), streamID, address.committed, got) + } + } + return nil +} + +// replayReadError says why a range a completed task recorded can no longer be +// read. Truncation and deletion are the reachable causes, and neither is +// recoverable for this workflow: without the bytes it can never replay, and +// without replaying it can never start another task. The bare read error names +// offsets and no workflow, which is not enough to act on. +func replayReadError( + consumer definition.WorkflowKey, + recorded *streampb.StreamCursor, + cause error, +) error { + return serviceerror.NewFailedPreconditionf( + "workflow %q cannot replay: stream %q no longer holds offsets [%d,%d) that one of its completed tasks consumed (%v)", + consumer.GetWorkflowID(), recorded.GetStreamId(), + recorded.GetFromOffset(), recorded.GetToOffset(), cause) +} + +// eventsOfResponse reads the events the response is carrying, whichever of the +// three representations it happens to be using. +func eventsOfResponse( + resp *historyservice.RecordWorkflowTaskStartedResponseWithRawHistory, +) ([]*historypb.HistoryEvent, error) { + if resp.GetHistory() != nil { + return resp.GetHistory().GetEvents(), nil + } + + raw := resp.GetRawHistoryBytes() + if len(raw) == 0 { + raw = resp.GetRawHistory() //nolint:staticcheck // SA1019: still populated while the newer field rolls out. + } + if len(raw) == 0 { + return nil, nil + } + + serializer := serialization.NewSerializer() + var events []*historypb.HistoryEvent + for _, batch := range raw { + batchEvents, err := serializer.DeserializeEvents(&commonpb.DataBlob{ + EncodingType: enumspb.ENCODING_TYPE_PROTO3, + Data: batch, + }) + if err != nil { + return nil, err + } + events = append(events, batchEvents...) + } + return events, nil +} diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go index a569e0ecc26..61520b6330f 100644 --- a/service/history/api/respondworkflowtaskcompleted/api.go +++ b/service/history/api/respondworkflowtaskcompleted/api.go @@ -452,6 +452,28 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( return nil, err } + // Subscriptions to streams in other executions, resolved here for the + // same reason: the command handler has nowhere to look the addressing + // up from, and by delivery time the cursor has to already exist. + // + // Skipped once the task has failed. Registering a consumer is a durable + // write on the stream's own execution, and this workflow's side of it + // is about to be rolled back, which would leave a pin on someone else's + // stream with no cursor behind it and nothing to release it. A failed + // command does not return an error, so this has to be checked here + // rather than inferred from err. + if workflowTaskHandler.workflowTaskFailedCause == nil && !workflowTaskHandler.stopProcessing { + if err = resolveStagedStreamSubscriptions( + ctx, + ms, + ms.GetWorkflowKey().NamespaceID, + completedEvent.GetEventId(), + workflowTaskHandler.stagedStreamSubscriptions, + ); err != nil { + return nil, err + } + } + // Worker must respond with Update Accepted or Update Rejected message on every Update Requested // message that were delivered on specific WT, when completing this WT. // If worker ignored the update request (old SDK or SDK bug), then server rejects this update. @@ -805,6 +827,14 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( // sticky is always enabled when worker request for new workflow task from RespondWorkflowTaskCompleted resp.StartedResponse.StickyExecutionEnabled = true + // The poll path delivers from its own handler, so this one has to ask + // as well or a subscribed workflow gets an inline task with no data. + resp.StartedResponse.StreamSlices, err = recordworkflowtaskstarted.DeliverStreamSlices( + ctx, handler.shardContext, ms) + if err != nil { + return nil, err + } + resp.NewWorkflowTask, err = handler.withNewWorkflowTask(ctx, namespaceEntry.Name(), req, resp.StartedResponse) if err != nil { return nil, err @@ -977,6 +1007,7 @@ func (handler *WorkflowTaskCompletedHandler) createPollWorkflowTaskQueueResponse StartedTime: matchingResp.StartedTime, Queries: matchingResp.Queries, Messages: matchingResp.Messages, + StreamSlices: matchingResp.StreamSlices, } return resp, nil diff --git a/service/history/api/respondworkflowtaskcompleted/stream_appends.go b/service/history/api/respondworkflowtaskcompleted/stream_appends.go new file mode 100644 index 00000000000..de783932a95 --- /dev/null +++ b/service/history/api/respondworkflowtaskcompleted/stream_appends.go @@ -0,0 +1,117 @@ +package respondworkflowtaskcompleted + +import ( + "context" + + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/stream" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + historyi "go.temporal.io/server/service/history/interfaces" +) + +// resolveStagedStreamSubscriptions turns subscribe commands for streams in +// other executions into cursors on this workflow. +// +// The lookup cannot happen in the command handler, which runs under the state +// lock with nowhere to do I/O from, and it cannot happen at delivery either, +// because by then the cursor has to already exist. So it happens here, between +// the commands and the commit. +// +// The pin goes on the stream before the cursor goes on the workflow, and that +// order is the guarantee. Interrupted after the first write there is a pin +// holding storage nothing reads, which costs space. The other order would leave +// a cursor no truncation floor protects, and truncation would be free to take a +// range it still points at. +func resolveStagedStreamSubscriptions( + ctx context.Context, + ms historyi.MutableState, + namespaceID string, + completedEventID int64, + staged []chasmworkflow.PendingStreamSubscription, +) error { + if len(staged) == 0 { + return nil + } + + wf, chasmCtx, err := ms.ChasmWorkflowComponent(ctx) + if err != nil { + return err + } + + for _, pending := range staged { + // Already subscribed, so only the event is owed. Re-registering would + // re-run the pin write with the original start offset, which would drag + // the stream's truncation floor back to where this consumer began. + if pending.AlreadySubscribed { + cursor, ok := wf.StreamCursors[pending.StreamID] + if !ok { + return serviceerror.NewInternalf( + "stream %q was marked already subscribed but has no cursor", pending.StreamID) + } + chasmworkflow.RecordStreamSubscribedOffset( + pending.Event, cursor.Get(chasmCtx).Offset()) + continue + } + + // A stream this workflow owns needs no lookup and no pin registration: + // it is in this execution, and its cursor commits with everything else. + if _, owned := wf.Streams[pending.StreamID]; owned { + startOffset, err := wf.SubscribeToOwnedStream( + chasmCtx, pending.StreamID, pending.StartOffset) + if err != nil { + return err + } + chasmworkflow.RecordStreamSubscribedOffset(pending.Event, startOffset) + continue + } + + ref := chasm.NewComponentRef[*stream.Stream](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: pending.StreamID, + }) + + // The engine rides the request context, installed by the interceptor. + state, err := chasm.ReadComponent(ctx, ref, (*stream.Stream).Snapshot, struct{}{}) + if err != nil { + return err + } + + startOffset := pending.StartOffset + if startOffset < 0 { + // Resolved once, here, and recorded. Left to delivery it would be a + // reading rather than a fact, and replay would resolve it again + // against a stream that has since moved. + startOffset = state.GetHeadOffset() + } + + consumerID := "workflow:" + ms.GetExecutionInfo().GetWorkflowId() + if _, _, err := chasm.UpdateComponent( + ctx, ref, + func(s *stream.Stream, mctx chasm.MutableContext, offset int64) (struct{}, error) { + return struct{}{}, s.RegisterConsumer( + mctx, consumerID, ms.GetExecutionInfo().GetWorkflowId(), "", offset, true) + }, + startOffset, + ); err != nil { + return err + } + + if _, err := wf.SubscribeToExternalStream(chasmCtx, chasmworkflow.ExternalStreamSubscription{ + StreamID: pending.StreamID, + CollectionID: state.GetCollectionId(), + BucketSize: state.GetBucketSize(), + StartOffset: startOffset, + KnownHead: state.GetHeadOffset(), + }); err != nil { + return err + } + + // Completed after the cursor exists. The event was written where the + // command was, but nothing outside this transaction sees either until + // the commit below, so a crash in between leaves no event claiming a + // subscription that was never made. + chasmworkflow.RecordStreamSubscribedOffset(pending.Event, startOffset) + } + return nil +} diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index 4e8da517eae..bb7e266681e 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -61,6 +61,9 @@ type ( workflowTaskDeployment *deploymentpb.Deployment // internal state + // Log writes staged by stream commands, flushed before this workflow + // task commits. + stagedStreamSubscriptions []chasmworkflow.PendingStreamSubscription hasBufferedEventsOrMessages bool workflowTaskFailedCause *workflowTaskFailedCause activityNotStartedCancelled bool @@ -354,6 +357,12 @@ func (handler *workflowTaskCompletedHandler) handleCommand( return nil, chasmErr } err = chasmHandler(chasmCtx, chasmWorkflow, validator, command, handlerOpts) + // A subscribe to a stream in another execution stages itself, + // since a command handler holds the state lock and has no + // context for the lookup it needs. Collect them for the + // resolution that has to precede this task's commit. + handler.stagedStreamSubscriptions = append( + handler.stagedStreamSubscriptions, chasmWorkflow.DrainStreamSubscriptions()...) // Fall back to the HSM handler either when the command type is not supported by CHASM (disabled // feature flag) or when the targeted entity is not owned by the CHASM tree (e.g. an operation // scheduled in HSM before the flag was flipped on). diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go index 501cf4dc51e..d0293e1a2dc 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go @@ -81,6 +81,9 @@ func TestCommandProtocolMessage(t *testing.T) { out.ms.EXPECT().VisitUpdates(gomock.Any()).AnyTimes() out.ms.EXPECT().GetNamespaceEntry().Return(tests.LocalNamespaceEntry).AnyTimes() out.ms.EXPECT().GetCurrentVersion().Return(tests.LocalNamespaceEntry.FailoverVersion(tests.WorkflowID)).AnyTimes() + out.ms.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + WorkflowTaskAttempt: 1, + }).AnyTimes() dcClient := dynamicconfig.StaticClient(nil) if opts.chasmEnabled { diff --git a/service/history/fx.go b/service/history/fx.go index dcf233185fe..0b004c3ab3e 100644 --- a/service/history/fx.go +++ b/service/history/fx.go @@ -11,6 +11,7 @@ import ( "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/chasm/lib/scheduler" + chasmstream "go.temporal.io/server/chasm/lib/stream/service" chasmtests "go.temporal.io/server/chasm/lib/tests" chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" @@ -123,6 +124,7 @@ var Module = fx.Options( hsmnexusoperations.Module, fx.Invoke(hsmnexusworkflow.RegisterCommandHandlers), activity.HistoryModule, + chasmstream.HistoryModule, scheduler.Module, callback.Module, chasmnexus.Module, diff --git a/service/history/historybuilder/event_factory.go b/service/history/historybuilder/event_factory.go index 058cb55b4ed..4f2c2f059af 100644 --- a/service/history/historybuilder/event_factory.go +++ b/service/history/historybuilder/event_factory.go @@ -10,6 +10,7 @@ import ( failurepb "go.temporal.io/api/failure/v1" historypb "go.temporal.io/api/history/v1" sdkpb "go.temporal.io/api/sdk/v1" + streampb "go.temporal.io/api/stream/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" updatepb "go.temporal.io/api/update/v1" workflowpb "go.temporal.io/api/workflow/v1" @@ -162,6 +163,7 @@ func (b *EventFactory) CreateWorkflowTaskCompletedEvent( deploymentName string, deployment *deploymentpb.Deployment, behavior enumspb.VersioningBehavior, + streamCursors []*streampb.StreamCursor, ) *historypb.HistoryEvent { event := b.createHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED, b.timeSource.Now()) event.Attributes = &historypb.HistoryEvent_WorkflowTaskCompletedEventAttributes{ @@ -176,6 +178,9 @@ func (b *EventFactory) CreateWorkflowTaskCompletedEvent( WorkerDeploymentName: deploymentName, DeploymentVersion: worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(deployment), VersioningBehavior: behavior, + // Offsets only. The payloads the task consumed rode the task + // response, so History grows with tasks rather than with messages. + StreamCursors: streamCursors, }, } diff --git a/service/history/historybuilder/event_store.go b/service/history/historybuilder/event_store.go index a6fb18a4b9e..ceb5d7b1d96 100644 --- a/service/history/historybuilder/event_store.go +++ b/service/history/historybuilder/event_store.go @@ -331,7 +331,9 @@ func (b *EventStore) bufferEvent( enumspb.EVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES, enumspb.EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED, - enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUESTED: + enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUESTED, + enumspb.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED, + enumspb.EVENT_TYPE_WORKFLOW_STREAM_MESSAGES_ADDED: // do not buffer event if event is directly generated from a corresponding command return false diff --git a/service/history/historybuilder/history_builder.go b/service/history/historybuilder/history_builder.go index 4690a0e92b1..85934808916 100644 --- a/service/history/historybuilder/history_builder.go +++ b/service/history/historybuilder/history_builder.go @@ -10,6 +10,7 @@ import ( failurepb "go.temporal.io/api/failure/v1" historypb "go.temporal.io/api/history/v1" sdkpb "go.temporal.io/api/sdk/v1" + streampb "go.temporal.io/api/stream/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" updatepb "go.temporal.io/api/update/v1" workflowpb "go.temporal.io/api/workflow/v1" @@ -236,6 +237,7 @@ func (b *HistoryBuilder) AddWorkflowTaskCompletedEvent( deploymentName string, deployment *deploymentpb.Deployment, behavior enumspb.VersioningBehavior, + streamCursors []*streampb.StreamCursor, ) *historypb.HistoryEvent { event := b.CreateWorkflowTaskCompletedEvent( scheduledEventID, @@ -248,6 +250,7 @@ func (b *HistoryBuilder) AddWorkflowTaskCompletedEvent( deploymentName, deployment, behavior, + streamCursors, ) event, _ = b.add(event) return event diff --git a/service/history/historybuilder/history_builder_categorization_test.go b/service/history/historybuilder/history_builder_categorization_test.go index 228824a50e5..43b6aed58e0 100644 --- a/service/history/historybuilder/history_builder_categorization_test.go +++ b/service/history/historybuilder/history_builder_categorization_test.go @@ -1277,6 +1277,7 @@ func (s *sutTestingAdapter) AddWorkflowTaskCompletedEvent(_ ...eventConfig) *his "", nil, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, + nil, ) } diff --git a/service/history/historybuilder/history_builder_test.go b/service/history/historybuilder/history_builder_test.go index d7b13d5fd6c..a8b02c4eaf8 100644 --- a/service/history/historybuilder/history_builder_test.go +++ b/service/history/historybuilder/history_builder_test.go @@ -704,6 +704,7 @@ func (s *historyBuilderSuite) TestWorkflowTaskCompleted() { "", nil, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, + nil, ) s.Equal(event, s.flush()) protorequire.ProtoEqual(s.T(), &historypb.HistoryEvent{ @@ -2272,6 +2273,8 @@ func (s *historyBuilderSuite) TestBufferEvent() { enumspb.EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED: true, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED: true, enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUESTED: true, + enumspb.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED: true, + enumspb.EVENT_TYPE_WORKFLOW_STREAM_MESSAGES_ADDED: true, } // events corresponding to message from client will be assigned an event ID immediately @@ -2321,7 +2324,8 @@ func (s *historyBuilderSuite) TestBufferEvent() { commandType := enumspb.CommandType(ct) // Unspecified is not counted. // ProtocolMessage command doesn't have corresponding event. - if commandType == enumspb.COMMAND_TYPE_UNSPECIFIED || commandType == enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE { + if commandType == enumspb.COMMAND_TYPE_UNSPECIFIED || + commandType == enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE { continue } commandsWithEventsCount++ diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index 632c0fbfb38..b806c4097d0 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -366,6 +366,15 @@ type ( ChasmEnabled() bool ChasmSignalBacklinksEnabled() bool ChasmWorkflowComponent(ctx context.Context) (*chasmworkflow.Workflow, chasm.MutableContext, error) + // HasChasmWorkflowComponent reports whether a workflow component is + // actually reachable. Archetype alone does not answer this: a workflow + // predating CHASM reports the workflow archetype while carrying a tree + // that holds no components. + HasChasmWorkflowComponent() bool + // HasPendingStreamData reports whether a stream subscription still has + // offsets to deliver, which is the only case where stream traffic + // schedules a workflow task. + HasPendingStreamData() bool ChasmWorkflowComponentReadOnly(ctx context.Context) (*chasmworkflow.Workflow, chasm.Context, error) // Ensures that the chasm workflow component is installed in the mutable state CHASM tree. // Must be called before adding any components to the tree. diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index 15a5b7221da..4cf77df6e5d 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -3052,6 +3052,20 @@ func (mr *MockMutableStateMockRecorder) HasBufferedEvents() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasBufferedEvents", reflect.TypeOf((*MockMutableState)(nil).HasBufferedEvents)) } +// HasChasmWorkflowComponent mocks base method. +func (m *MockMutableState) HasChasmWorkflowComponent() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasChasmWorkflowComponent") + ret0, _ := ret[0].(bool) + return ret0 +} + +// HasChasmWorkflowComponent indicates an expected call of HasChasmWorkflowComponent. +func (mr *MockMutableStateMockRecorder) HasChasmWorkflowComponent() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasChasmWorkflowComponent", reflect.TypeOf((*MockMutableState)(nil).HasChasmWorkflowComponent)) +} + // HasCompletedAnyWorkflowTask mocks base method. func (m *MockMutableState) HasCompletedAnyWorkflowTask() bool { m.ctrl.T.Helper() @@ -3080,6 +3094,20 @@ func (mr *MockMutableStateMockRecorder) HasParentExecution() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasParentExecution", reflect.TypeOf((*MockMutableState)(nil).HasParentExecution)) } +// HasPendingStreamData mocks base method. +func (m *MockMutableState) HasPendingStreamData() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasPendingStreamData") + ret0, _ := ret[0].(bool) + return ret0 +} + +// HasPendingStreamData indicates an expected call of HasPendingStreamData. +func (mr *MockMutableStateMockRecorder) HasPendingStreamData() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasPendingStreamData", reflect.TypeOf((*MockMutableState)(nil).HasPendingStreamData)) +} + // HasPendingWorkflowTask mocks base method. func (m *MockMutableState) HasPendingWorkflowTask() bool { m.ctrl.T.Helper() diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 069c3e37a24..c1c0d277c6c 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -22,6 +22,7 @@ import ( historypb "go.temporal.io/api/history/v1" rulespb "go.temporal.io/api/rules/v1" "go.temporal.io/api/serviceerror" + streampb "go.temporal.io/api/stream/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" updatepb "go.temporal.io/api/update/v1" workerpb "go.temporal.io/api/worker/v1" @@ -672,6 +673,98 @@ func (ms *MutableStateImpl) mustInitHSM() { ms.stateMachineNode = stateMachineNode } +// commitStreamCursors folds each staged range into its cursor and returns the +// ranges to record. Called as the completed event is built, so the advance and +// the event carrying the range are in one transaction: split apart, a crash +// between them would either redeliver a range or skip it with nothing in +// History to say so. +func (ms *MutableStateImpl) commitStreamCursors() ([]*streampb.StreamCursor, error) { + if !ms.HasChasmWorkflowComponent() { + return nil, nil + } + // Reaching the component through a mutable context marks it dirty, which + // would add a node to the transaction of every workflow that has no + // subscription at all. Ask read-only first, and take the write path only + // when there is something to fold in. + // + // The background context matches EnsureChasmWorkflowComponent: it only + // reaches components already in memory. + wf, _, err := ms.ChasmWorkflowComponentReadOnly(context.Background()) + if err != nil { + return nil, err + } + if len(wf.StreamCursors) == 0 { + return nil, nil + } + + wf, chasmCtx, err := ms.ChasmWorkflowComponent(context.Background()) + if err != nil { + return nil, err + } + return wf.CommitStreamCursors(chasmCtx), nil +} + +// carryStreamSubscriptionsTo hands this run's subscriptions to the run that +// continues it. +// +// A cursor is workflow state, so without this a continue-as-new would silently +// end a subscription the workflow never cancelled: the stream would keep its +// consumer pin and keep pushing to the workflow id, and the successor would +// have nowhere to put it. +func (ms *MutableStateImpl) carryStreamSubscriptionsTo(newMutableState *MutableStateImpl) error { + if !ms.HasChasmWorkflowComponent() { + return nil + } + wf, chasmCtx, err := ms.ChasmWorkflowComponentReadOnly(context.Background()) + if err != nil { + return err + } + subscriptions := wf.ExportStreamSubscriptions(chasmCtx) + if len(subscriptions) == 0 { + return nil + } + + newMutableState.EnsureChasmWorkflowComponent(context.Background()) + newWorkflow, newChasmCtx, err := newMutableState.ChasmWorkflowComponent(context.Background()) + if err != nil { + return err + } + return newWorkflow.ImportStreamSubscriptions(newChasmCtx, subscriptions) +} + +// HasPendingStreamData reports whether a subscription of this workflow has +// offsets left to deliver, which is the one condition under which stream +// traffic schedules a workflow task. +// +// Called on every transaction close for every workflow, almost none of which +// have a subscription, so it resolves the root component once rather than +// asking whether it exists and then asking for it. +func (ms *MutableStateImpl) HasPendingStreamData() bool { + node, ok := ms.chasmTree.(*chasm.Node) + if !ok { + return false + } + chasmCtx := chasm.NewContext(context.Background(), node) + rootComponent, err := node.ComponentByPath(chasmCtx, nil) + if err != nil { + return false + } + wf, ok := rootComponent.(*chasmworkflow.Workflow) + if !ok { + return false + } + return wf.StreamCursorsBehind(chasmCtx) +} + +func (ms *MutableStateImpl) HasChasmWorkflowComponent() bool { + node, ok := ms.chasmTree.(*chasm.Node) + if !ok { + return false + } + _, err := node.ComponentByPath(chasm.NewContext(context.Background(), node), nil) + return err == nil +} + func (ms *MutableStateImpl) IsWorkflow() bool { return ms.chasmTree.ArchetypeID() == chasm.WorkflowArchetypeID } @@ -6394,6 +6487,10 @@ func (ms *MutableStateImpl) AddContinueAsNewEvent( return nil, nil, err } + if err = ms.carryStreamSubscriptionsTo(newMutableState); err != nil { + return nil, nil, err + } + if err = ms.ApplyWorkflowExecutionContinuedAsNewEvent( batchID, continueAsNewEvent, @@ -7939,6 +8036,25 @@ func (ms *MutableStateImpl) closeTransactionHandleWorkflowTaskScheduling( } } + // A stream subscription with offsets left to deliver. Handled here rather + // than at workflow task completion so it also covers the transaction that + // registers a subscription against a stream that already has data, which + // completes no workflow task of its own. + // + // The pending-task check comes first because it is cheap: a workflow that + // already owes a task needs no further reason to run, so the subscription + // state is only consulted when the answer could change something. + if !ms.HasPendingWorkflowTask() && + !ms.IsWorkflowExecutionStatusPaused() && + ms.HasPendingStreamData() { + if _, err := ms.AddWorkflowTaskScheduledEvent( + false, + enumsspb.WORKFLOW_TASK_TYPE_NORMAL, + ); err != nil { + return err + } + } + return nil } diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go index 72123c71bfa..9ecc63b75ae 100644 --- a/service/history/workflow/workflow_task_state_machine.go +++ b/service/history/workflow/workflow_task_state_machine.go @@ -826,6 +826,11 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskCompletedEvent( //nolint:staticcheck // SA1019 deprecated Deployment will clean up later wftDeployment := worker_versioning.DeploymentOrVersion(request.Deployment, wftDeploymentVersion) + streamCursors, err := m.ms.commitStreamCursors() + if err != nil { + return nil, err + } + // Now write the completed event event := m.ms.hBuilder.AddWorkflowTaskCompletedEvent( workflowTask.ScheduledEventID, @@ -838,6 +843,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskCompletedEvent( deploymentName, wftDeployment, vb, + streamCursors, ) override := m.ms.GetExecutionInfo().GetVersioningInfo().GetVersioningOverride() @@ -849,7 +855,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskCompletedEvent( } wftScheduleToClose := event.GetEventTime().AsTime().Sub(workflowTask.ScheduledTime) - err := m.afterAddWorkflowTaskCompletedEvent(event, limits, wftScheduleToClose) + err = m.afterAddWorkflowTaskCompletedEvent(event, limits, wftScheduleToClose) if err != nil { return nil, err } diff --git a/service/matching/matching_engine.go b/service/matching/matching_engine.go index 049da260866..bb60c55ce42 100644 --- a/service/matching/matching_engine.go +++ b/service/matching/matching_engine.go @@ -3367,6 +3367,7 @@ func (e *matchingEngineImpl) convertPollWorkflowTaskQueueResponse( StartedTime: resp.StartedTime, Queries: resp.Queries, Messages: resp.Messages, + StreamSlices: resp.StreamSlices, History: history, NextPageToken: resp.NextPageToken, PollerScalingDecision: resp.PollerScalingDecision, diff --git a/service/matching/wire_compat_test.go b/service/matching/wire_compat_test.go new file mode 100644 index 00000000000..56fcfa06524 --- /dev/null +++ b/service/matching/wire_compat_test.go @@ -0,0 +1,173 @@ +package matching + +import ( + "fmt" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/matchingservice/v1" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// Each started-task response exists twice, once carrying History and once +// carrying raw bytes, so matching can forward a task without deserializing it. +// The two are only interchangeable while every field number means the same +// thing on both sides. That invariant is stated in a proto comment, which is +// easy to satisfy on one copy and forget on the other, and a mismatch stays +// invisible until history.sendRawHistoryBetweenInternalServices is enabled. +func TestStartedResponsePairsStayWireCompatible(t *testing.T) { + t.Run("history service", func(t *testing.T) { + // raw_history is a History on one side and repeated bytes on the other, + // which is the whole point of keeping two messages. + require.Empty(t, wireCompatDiff( + &historyservice.RecordWorkflowTaskStartedResponse{}, + &historyservice.RecordWorkflowTaskStartedResponseWithRawHistory{}, + 20, + )) + }) + + t.Run("matching service", func(t *testing.T) { + require.Empty(t, wireCompatDiff( + &matchingservice.PollWorkflowTaskQueueResponse{}, + &matchingservice.PollWorkflowTaskQueueResponseWithRawHistory{}, + 22, + )) + }) +} + +// Without a negative control the check above passes just as happily when it has +// stopped inspecting anything. Drop the exemption and the one field the pair is +// allowed to disagree on has to be reported. +func TestWireCompatDiffDetectsAMismatch(t *testing.T) { + diff := wireCompatDiff( + &historyservice.RecordWorkflowTaskStartedResponse{}, + &historyservice.RecordWorkflowTaskStartedResponseWithRawHistory{}, + ) + require.Len(t, diff, 1) + require.Contains(t, diff[0], "field 20") + require.Contains(t, diff[0], "raw_history") +} + +// wireCompatDiff reports every way the two messages would disagree on the wire, +// ignoring the field numbers named in typeMayDiffer. +func wireCompatDiff( + a proto.Message, + b proto.Message, + typeMayDiffer ...protoreflect.FieldNumber, +) []string { + exempt := make(map[protoreflect.FieldNumber]struct{}, len(typeMayDiffer)) + for _, number := range typeMayDiffer { + exempt[number] = struct{}{} + } + + aName := a.ProtoReflect().Descriptor().Name() + bName := b.ProtoReflect().Descriptor().Name() + aFields := fieldsByNumber(a) + bFields := fieldsByNumber(b) + + var problems []string + report := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf(format, args...)) + } + + for _, number := range sortedFieldNumbers(aFields) { + aField := aFields[number] + bField, ok := bFields[number] + if !ok { + report("field %d (%s) is on %s but missing from %s", number, aField.Name(), aName, bName) + continue + } + if aField.Name() != bField.Name() { + report("field %d is %s on %s and %s on %s", number, aField.Name(), aName, bField.Name(), bName) + continue + } + if _, ok := exempt[number]; ok { + continue + } + + // A map field's value type is a synthesized entry message nested in its + // parent, so its full name always differs across the pair. Compare what + // actually goes on the wire instead. + if aField.IsMap() { + if !bField.IsMap() { + report("field %d (%s) is a map on %s only", number, aField.Name(), aName) + continue + } + if aField.MapKey().Kind() != bField.MapKey().Kind() { + report("field %d (%s) has a different map key type on each side", number, aField.Name()) + } + if problem := valueTypeDiff(aField.MapValue(), bField.MapValue(), number, aField.Name()); problem != "" { + report("%s", problem) + } + continue + } + + if aField.Cardinality() != bField.Cardinality() { + report("field %d (%s) is %s on %s and %s on %s", + number, aField.Name(), aField.Cardinality(), aName, bField.Cardinality(), bName) + continue + } + if problem := valueTypeDiff(aField, bField, number, aField.Name()); problem != "" { + report("%s", problem) + } + } + + for _, number := range sortedFieldNumbers(bFields) { + if _, ok := aFields[number]; !ok { + report("field %d (%s) is on %s but missing from %s", number, bFields[number].Name(), bName, aName) + } + } + + return problems +} + +func valueTypeDiff( + a protoreflect.FieldDescriptor, + b protoreflect.FieldDescriptor, + number protoreflect.FieldNumber, + name protoreflect.Name, +) string { + if a.Kind() != b.Kind() { + return fmt.Sprintf("field %d (%s) is %s on one side and %s on the other", number, name, a.Kind(), b.Kind()) + } + + switch a.Kind() { + case protoreflect.MessageKind, protoreflect.GroupKind: + if a.Message().FullName() != b.Message().FullName() { + return fmt.Sprintf("field %d (%s) refers to %s on one side and %s on the other", + number, name, a.Message().FullName(), b.Message().FullName()) + } + case protoreflect.EnumKind: + if a.Enum().FullName() != b.Enum().FullName() { + return fmt.Sprintf("field %d (%s) refers to %s on one side and %s on the other", + number, name, a.Enum().FullName(), b.Enum().FullName()) + } + default: + // Scalars are fully described by the kind equality checked above. + } + return "" +} + +func fieldsByNumber(m proto.Message) map[protoreflect.FieldNumber]protoreflect.FieldDescriptor { + fields := m.ProtoReflect().Descriptor().Fields() + byNumber := make(map[protoreflect.FieldNumber]protoreflect.FieldDescriptor, fields.Len()) + for i := range fields.Len() { + field := fields.Get(i) + byNumber[field.Number()] = field + } + return byNumber +} + +// Field order drives the order of reported problems, which keeps a failure +// message stable between runs. +func sortedFieldNumbers(fields map[protoreflect.FieldNumber]protoreflect.FieldDescriptor) []protoreflect.FieldNumber { + numbers := make([]protoreflect.FieldNumber, 0, len(fields)) + for number := range fields { + numbers = append(numbers, number) + } + slices.Sort(numbers) + return numbers +} diff --git a/service/worker/scanner/history/scavenger.go b/service/worker/scanner/history/scavenger.go index bc0543a18d1..a2cedbad82e 100644 --- a/service/worker/scanner/history/scavenger.go +++ b/service/worker/scanner/history/scavenger.go @@ -216,6 +216,18 @@ func (s *Scavenger) filterTask( return nil } + // Not a workflow execution's history, so its owner deletes it and this + // scavenger must not. Checked before the parse, because the tag parses + // into a namespace that does not exist and that reads as garbage. + if persistence.IsNonExecutionGarbageCleanupInfo(branch.Info) { + metrics.HistoryScavengerSkipCount.With(s.metricsHandler).Record(1) + + s.Lock() + defer s.Unlock() + s.hbd.SkipCount++ + return nil + } + namespaceID, workflowID, runID, err := persistence.SplitHistoryGarbageCleanupInfo(branch.Info) if err != nil { s.logger.Error("unable to parse the history cleanup info", tag.DetailInfo(branch.Info), tag.Error(err)) diff --git a/service/worker/scanner/history/scavenger_test.go b/service/worker/scanner/history/scavenger_test.go index d9e7e45e52c..5a12ea5a375 100644 --- a/service/worker/scanner/history/scavenger_test.go +++ b/service/worker/scanner/history/scavenger_test.go @@ -744,3 +744,61 @@ func (s *ScavengerTestSuite) TestDeleteWorkflowAfterRetention() { s.Equal(2, hbd.CurrentPage) s.Equal(0, len(hbd.NextPageToken)) } + +// A branch that is not a workflow execution's history must survive the +// scavenger. It deletes a branch whose execution it cannot find, and a stream +// log has no execution to find, so without the marker it reclaims live data +// once the branch passes the minimum age. +func (s *ScavengerTestSuite) TestSkipsBranchesThatAreNotExecutionHistory() { + streamInfo := persistence.NonExecutionGarbageCleanupInfoPrefix + "stream:namespaceID1:collection1" + + s.mockExecutionManager.EXPECT().GetAllHistoryTreeBranches(gomock.Any(), protomock.Eq(&persistence.GetAllHistoryTreeBranchesRequest{ + PageSize: pageSize, + })).Return(&persistence.GetAllHistoryTreeBranchesResponse{ + Branches: []persistence.HistoryBranchDetail{ + { + BranchInfo: &persistencespb.HistoryBranch{ + TreeId: treeID1, + BranchId: branchID1, + }, + ForkTime: timestamp.TimeNowPtrUtcAddDuration(-s.scavenger.historyDataMinAge() * 2), + Info: streamInfo, + }, + { + BranchInfo: &persistencespb.HistoryBranch{ + TreeId: treeID2, + BranchId: branchID2, + }, + ForkTime: timestamp.TimeNowPtrUtcAddDuration(-s.scavenger.historyDataMinAge() * 2), + Info: persistence.BuildHistoryGarbageCleanupInfo("namespaceID2", "workflowID2", "runID2"), + }, + }, + }, nil) + + // Only the workflow branch is looked up, and only it is deleted. The stream + // branch is never described, because describing it is what produced the + // not-found the scavenger read as garbage. + s.mockHistoryClient.EXPECT().DescribeMutableState(gomock.Any(), &historyservice.DescribeMutableStateRequest{ + NamespaceId: "namespaceID2", + Execution: &commonpb.WorkflowExecution{ + WorkflowId: "workflowID2", + RunId: "runID2", + }, + ArchetypeId: chasm.WorkflowArchetypeID, + }).Return(nil, serviceerror.NewNotFound("")) + branchToken2, err := s.scavenger.serializer.HistoryBranchToBlob(&persistencespb.HistoryBranch{ + TreeId: treeID2, + BranchId: branchID2, + }) + s.Require().NoError(err) + s.mockExecutionManager.EXPECT().DeleteHistoryBranch(gomock.Any(), protomock.Eq(&persistence.DeleteHistoryBranchRequest{ + ShardID: common.WorkflowIDToHistoryShard("namespaceID2", "workflowID2", s.scavenger.numShards), + BranchToken: branchToken2.Data, + })).Return(nil) + + hbd, err := s.scavenger.Run(context.Background()) + s.Require().NoError(err) + s.Equal(1, hbd.SkipCount, "the stream branch must be skipped, not collected") + s.Equal(1, hbd.SuccessCount) + s.Equal(0, hbd.ErrorCount) +} diff --git a/streaming-benchmark-results.md b/streaming-benchmark-results.md new file mode 100644 index 00000000000..b6112578f95 --- /dev/null +++ b/streaming-benchmark-results.md @@ -0,0 +1,119 @@ +# Native Streams: measured against today's Workflow Streams + +| | | +|---|---| +| Status | First measurement, single run per cell | +| Ticket | [AI-198](https://temporalio.atlassian.net/browse/AI-198) | +| Harness | `tests/streaming_baseline_test.go`, `tests/streaming_native_test.go` | +| Date | 2026-08-24 | + +Both designs over the same workload. The two halves deliberately share workload generation, latency stamping, and metric capture, because a benchmark whose halves generate load differently measures the harness rather than the design. + +## The result in one line + +At 100ms batching with 25 subscribers, the current pattern delivers **19%** of expected messages with a **11.2 second** p99. The native path delivers **100%** with a **104ms** p99, at **1/20th** the persistence cost per message. + +## Method + +The shipped pattern reproduced end to end: a producer generates messages continuously and flushes them into the workflow as batched **Signals**; consumers read them back with a long-polling **Update**; the workflow holds the buffer. + +- 40 messages/sec of 20 bytes, 50 seconds, so about 2000 messages per cell. +- Flush intervals 2s (the shipped default) and 100ms (the bar the Native Streams 1-pager sets). +- 1, 5, and 25 concurrent subscribers. +- Dedicated single-node cluster, SQLite, namespace-scoped `persistence_requests`. +- Latency is stamped at **generation**, not at flush. Stamping at flush measures only the server round trip and hides the batching delay, which is the dominant term. +- Persistence counts exclude cluster, namespace, and workflow start, so they reflect streaming steady state. + +Reproduce with `TEMPORAL_STREAM_BENCH=1 go test -tags test_dep ./tests/ -run TestStreamingComparison -v -timeout 45m`. Without the variable a two-cell short version runs, which keeps the harness from rotting. + +## Head to head + +| scenario | design | msgs | delivered | rejected polls | wf history bytes/msg | persist ops/msg | p50 | p99 | +|---|---|---|---|---|---|---|---|---| +| 2s, 1 sub | signals+update | 2000 | 2000 | 2 | 71.37 | 0.08 | 1.010s | 1.985s | +| 2s, 1 sub | **native** | 2000 | 2000 | 1 | **0.00** | **0.03** | 0.981s | 1.976s | +| 2s, 5 subs | signals+update | 2000 | 10000 | 10 | 198.66 | 0.10 | 1.006s | 1.981s | +| 2s, 5 subs | **native** | 2000 | 10000 | 5 | **0.00** | **0.03** | 1.003s | 1.979s | +| 2s, 25 subs | signals+update | 2000 | 20000 | 4543 | 358.99 | 2.45 | 1.009s | 1.987s | +| 2s, 25 subs | **native** | 2000 | **50000** | 25 | **0.00** | **0.03** | 1.003s | 1.978s | +| 100ms, 1 sub | signals+update | 1999 | 1999 | 2 | 379.12 | 1.52 | 59ms | 108ms | +| 100ms, 1 sub | **native** | 2000 | 2000 | 1 | **0.00** | **0.50** | 52ms | 101ms | +| 100ms, 5 subs | signals+update | 1999 | 7995 | 2329 | 706.43 | 2.62 | 60ms | 110ms | +| 100ms, 5 subs | **native** | 2000 | **10000** | 5 | **0.00** | **0.51** | 54ms | 102ms | +| 100ms, 25 subs | signals+update | 2000 | 9560 | 19784 | 698.72 | 11.30 | 64ms | 11.212s | +| 100ms, 25 subs | **native** | 2000 | **50000** | 25 | **0.00** | **0.55** | 53ms | 104ms | + +Expected delivery is messages times subscribers. The native path reaches it in every cell; the current pattern reaches it only at low subscriber counts. + +### What the native cost is made of + +The per-message figures are not approximations of something complicated. At 2s batching over 2000 messages the native path performs 25 flushes, and the raw breakdown is `AppendRawHistoryNodes: 25` and `UpdateWorkflowExecution: 25`. One log append and one frontier update per flush, and **nothing at all for the reads**, no matter how many readers there are. + +That is the design's central claim, and it is why the cost per message does not move between 1 and 25 subscribers while the current pattern's rises from 1.52 to 11.30. + +## Baseline detail + +| scenario | msgs | delivered | rejected polls | hist events/msg | hist bytes/msg | persist ops/msg | p50 | p99 | +|---|---|---|---|---|---|---|---|---| +| flush 2s, 1 sub | 2000 | 2000 | 2 | 0.11 | 70.85 | 0.08 | 989ms | 1.985s | +| flush 2s, 5 subs | 2000 | 10000 | 10 | 0.22 | 198.15 | 0.10 | 1.008s | 1.986s | +| flush 2s, 25 subs | 1999 | 21830 | 4771 | 0.36 | 385.82 | 2.58 | 1.09s | 40.571s | +| flush 100ms, 1 sub | 1999 | 1999 | 2 | 2.25 | 373.64 | 1.53 | 57ms | 108ms | +| flush 100ms, 5 subs | 1999 | 8000 | 2328 | 3.62 | 699.73 | 2.61 | 62ms | 109ms | +| flush 100ms, 25 subs | 2000 | 15422 | 11621 | 3.13 | 735.60 | 6.91 | 108ms | 38.881s | + +"Delivered" counts message receipts across all subscribers, so the expected value is messages times subscribers. + +## What the baseline alone shows + +### Latency is bought with cost, at roughly 20x + +At one subscriber, tightening the flush from 2s to 100ms improves p50 latency 17x (989ms to 57ms) and costs: + +- **20x** more history events per message (0.11 to 2.25) +- **5.3x** more history bytes per message (70.9 to 373.6) +- **19x** more persistence operations per message (0.08 to 1.53) + +The p50 and p99 figures are a check on the method as much as a result: p50 lands at about half the flush interval and p99 at about the full interval, which is what a uniform batching delay must produce. If they had not, the measurement would have been wrong. + +### There is a hard ceiling at 10 concurrent subscribers + +`history.maxInFlightUpdates` defaults to **10** per workflow execution (`common/dynamicconfig/constants.go:2559`). At 25 subscribers the pattern delivers 44% of expected receipts with 4771 rejected polls and a p99 of 40 seconds, even at the cheap 2s flush. This is the "maximum of 10 concurrent subscribers" limit from the 1-pager, now with numbers attached: it does not degrade gracefully, it starves. + +### There is a second ceiling nobody has been talking about + +`history.maxTotalUpdates` defaults to **2000 per workflow execution** (`constants.go:2569`). Every poll consumes one, whether or not it returns anything. + +At 100ms batching a single subscriber burns about 500 polls in 50 seconds. Five subscribers exhaust a workflow's entire lifetime update budget in **under a minute**, which is exactly what the 100ms/5-subscriber cell shows: delivery stops dead at 8000 receipts with 2328 rejections following. + +The implication for agent sessions is the sharper form of the cost problem. A workflow is not merely paying per stream item; **it is spending a finite, non-renewable per-execution budget on the act of reading**. A long agent conversation would need continue-as-new every few minutes purely to reset a poll counter, which is a lifecycle event driven by the transport rather than by the application. + +This is the strongest argument in the data for moving reads off the workflow entirely. A design where reading costs no state transition does not have this ceiling at all, because there is nothing to exhaust. + +## Caveats + +- Single run per cell, no repetitions, so treat the numbers as an order of magnitude rather than a precise figure. +- SQLite on a single-node dev cluster. Cassandra behaviour, especially per-partition cost, is not addressed and these numbers must not be read as speaking to it. +- Persistence ops per message in the rejecting cells (2.58 and 6.91) are inflated by retry traffic from rejected polls. The uncontended cells (0.08 and 1.53) are the ones to quote for the cost comparison. +- The harness disables the test logger's failure-on-error behaviour, because a saturated cluster torn down mid-drain always logs shard-status errors. An anomalous result should be re-run with that off before it is trusted. +- Latency is measured from a simulated generation clock, not from a real LLM token stream. + +## Two measurement bugs worth recording + +Both produced confident, wrong numbers, and neither would have been caught by a test passing. + +**Latency was stamped at flush.** That measures only the server round trip and hides the batching delay, which is the dominant term and the entire reason a shorter interval is wanted. The corrected figures self-check: p50 lands at about half the flush interval and p99 at about the full interval, which is what a uniform batching delay must produce. + +**The native path's persistence operations were not being counted at all.** The first run reported 0.00 per message for a path that demonstrably writes on every append. Persistence metrics are keyed on caller info carried by the context, not on the request, and the stream wrote its log directly without setting it. That was a real bug rather than a harness artifact: those calls were also escaping namespace rate limiting and priority. Only the raw per-operation breakdown exposed it; the aggregate looked like a triumph. + +## What the numbers do not cover + +- Single run per cell, no repetitions. Treat them as order of magnitude. +- SQLite on a single-node dev cluster. Cassandra behaviour, especially per-partition cost, is not addressed and these numbers must not be read as speaking to it. +- Persistence ops per message in the rejecting baseline cells are inflated by retry traffic from rejected polls. +- The native path is measured with the producer and consumers off-shard, which is the path LLM token streaming takes. Publishing from inside a workflow is built since this run but not measured here. Consuming inside one is not built. +- Latency comes from a simulated generation clock, not a real LLM token stream. + +## Next + +The remaining gap between this and a production claim is Cassandra, replication, and the in-workflow paths. Nothing here depends on group commit, which stays deferred: the append path already costs two writes per flush without it. diff --git a/streaming-cassandra-partitioning.md b/streaming-cassandra-partitioning.md new file mode 100644 index 00000000000..a498a5a95f0 --- /dev/null +++ b/streaming-cassandra-partitioning.md @@ -0,0 +1,99 @@ +# Cassandra partitioning for the stream log + +Measured 2026-09-02 against Cassandra 5.0.9, single node, on the schema the +dedicated-facet substrate would use. This was named as the single biggest risk +to a Temporal-owned stream and had never been run. + +## What was measured + +The proposed table, one partition per bucket: + +```sql +CREATE TABLE stream_log ( + shard_id int, + namespace_id uuid, + collection_id text, + bucket bigint, + start_offset bigint, + end_offset bigint, + data blob, + data_encoding text, + PRIMARY KEY ((shard_id, namespace_id, collection_id, bucket), start_offset) +) WITH CLUSTERING ORDER BY (start_offset ASC); +``` + +One row is one appended batch, keyed by the offset its first message landed at. + +| shape | rows in the bucket | payload per row | partition | +|---|---|---|---| +| unbatched token stream | 100,000 | 20 B | 5.84 MB | +| lightly batched | 50,000 | 100 B | 4.87 MB | +| large batches | 50 | 512 KB | 30.13 MB | + +Sizes are `Compacted partition bytes` from `nodetool tablestats`, which is the +logical size. On-disk figures are not quoted: the synthetic payload is a +repeating byte pattern and compresses far better than text would. + +## What it says + +**Row count binds before size does, for the shape that matters.** An agent +session of 100k unbatched tokens fits in 5.84 MB, comfortably inside the 100 MB +guidance, but it is 100,000 rows in one partition, which is exactly the +rule-of-thumb ceiling. Bucket size therefore has to be chosen against the +unbatched case, because a bucket spans offsets and only the producer decides how +many offsets ride in a row. + +A bucket of 10,000 offsets gives about 580 KB and 10,000 rows unbatched, which +leaves room on both axes. That is the number to start from, not the current +100,000. + +**The current byte limit permits partitions that are too large.** A batch may +carry up to 1,000 messages, each up to the payload limit. At 2 KB per message +that is a 2 MB row, and a 100,000-offset bucket holds 100 of them: roughly +200 MB, twice the guidance, in cells that are themselves against Cassandra's +advice. The measured 512 KB rows already produce a 30 MB partition from 50 rows. + +So a bucket needs a byte budget as well as an offset budget, and must roll on +whichever is reached first. The current design has only the offset budget. This +is a design gap the measurement found, and it applies to the existing substrate +as much as to the proposed one. + +**Reads position in one row.** Tracing a read for an offset deep inside the +100,000-row partition: + +``` +SELECT ... WHERE AND start_offset <= 73456 ORDER BY start_offset DESC LIMIT 1 + -> Read 1 live rows and 0 tombstone cells +SELECT ... WHERE AND start_offset >= 73456 LIMIT 100 + -> Read 100 live rows and 0 tombstone cells +``` + +One row to find the batch containing an arbitrary offset, then exactly the page +asked for. Keying by the offset a batch starts at is what buys that. + +The current `history_node` substrate cannot do it. A node id is the first offset +of its batch, so a read starting inside a batch does not know where that batch +began and compensates by starting `MaxMessagesPerBatch` rows earlier +(`chasm/lib/stream/log.go`, `startNode := NodeIDOf(...) - MaxMessagesPerBatch + 1`). +That is up to 1,000 clustering rows read behind the target on every poll, for +every reader. + +## Conclusion + +Cassandra is not the obstacle it was feared to be, on either substrate, provided +buckets roll on bytes as well as offsets. What the measurement does show is a +read-amplification difference that favours the dedicated facet by three orders +of magnitude per poll, which is a second reason to move beyond the correctness +ones. + +## Reproducing + +`develop/cassmeasure/` has the generator and the read trace. Start Cassandra with +`docker compose -f develop/docker-compose/docker-compose.yml up -d cassandra`. + +## Caveats + +Single node, no replication, no compaction pressure, no concurrent writers. +Synthetic compressible payloads. Partition sizes are logical rather than on +disk. None of that changes the row counts or the read shapes, which is what the +conclusions rest on. diff --git a/streaming-detailed-design.md b/streaming-detailed-design.md new file mode 100644 index 00000000000..ec114c26a79 --- /dev/null +++ b/streaming-detailed-design.md @@ -0,0 +1,904 @@ +# Native Streams: Detailed Design + +| | | +|---|---| +| Status | Draft for review | +| Ticket | [AI-198](https://temporalio.atlassian.net/browse/AI-198) | +| Author | Moe Dashti | +| Date | 2026-08-23 | +| Companion | `streaming-high-level-design.md`, `design-comparison.md` | + +This document specifies the implementation. It assumes the high-level design and does not re-argue it. Line references are against `main` at `6805caea5`. + +Changes adopted from the two existing prototypes are folded in here; `design-comparison.md` §7 records which change came from where. + +--- + +## 1. Package layout + +New CHASM library, following `docs/architecture/chasm.md` and modelled on `chasm/lib/activity`. + +``` +chasm/lib/stream/ +├── proto/v1/ +│ ├── stream_state.proto # persisted component state +│ ├── message.proto # StreamMessage, StreamMessageBatch +│ ├── request_response.proto +│ ├── service.proto # StreamService +│ └── tasks.proto # close / retention tasks +├── gen/streampb/v1/ # generated; picked up by CHASM_PROTO_FILES (Makefile:105) +├── stream.go # Stream component and its transitions +├── log.go # branch mint, append, range read +├── tailcache.go # shard-local ring of recent batches +├── config.go +├── handler.go # history-side gRPC handler +├── frontend.go # namespace name to ID, forward via layered client +├── library.go +├── fx.go # Module (history) and FrontendModule +└── tasks.go +``` + +--- + +## 2. Wire types + +### 2.1 Item and batch + +```protobuf +// message.proto +message StreamMessage { + temporal.api.common.v1.Payload body = 1; + // Producer-supplied and, later, server-enriched provenance + // (workflow_id, run_id, original_run_id, attempt). Off by default. + map metadata = 2; + string topic = 3; + // Position within this topic. The global offset orders the whole stream; + // this lets a consumer reason about one topic without decoding the rest. + int64 topic_sequence = 4; + // Ordinary payload, or an in-band control marker such as a flush. + StreamMessageKind kind = 5; +} + +enum StreamMessageKind { + STREAM_MESSAGE_KIND_UNSPECIFIED = 0; + STREAM_MESSAGE_KIND_DATA = 1; + // Producer signalling a delivery boundary; see 4.1b. + STREAM_MESSAGE_KIND_FLUSH = 2; +} + +// One append is one batch, and one batch is one history node. +message StreamMessageBatch { + repeated StreamMessage messages = 1; +} +``` + +`StreamMessageBatch` is what gets serialized into the node blob. The server does not deserialize it on the normal read path; see §4.3 for the one case where it does. + +### 2.2 Component state + +```protobuf +// stream_state.proto +message StreamState { + bytes branch_token = 1; + + // Visibility frontier. Readers never observe an offset at or past this. + int64 head_offset = 2; + // Truncation floor. Offsets below this are gone. + int64 base_offset = 3; + // Chains history nodes; see AppendRawHistoryNodesRequest.PrevTransactionID. + int64 last_txn_id = 4; + + bool closed = 5; + temporal.api.common.v1.Payload close_reason = 6; + + // Bumped on ownership change so a stale producer's write fails. + int64 owner_epoch = 7; + + // Immutable once set. Offsets are bucketed into separate history-node trees + // so no Cassandra partition grows with the stream; see 3.1a. + int64 bucket_size = 11; + + // Set when a successor run takes ownership, so an in-flight poll can follow + // the chain instead of stalling on a superseded run; see 9a. + string redirect_run_id = 12; + + // producer_id -> last accepted (seq, first_offset). Bounded by producer count. + map producers = 8; + // Registered in-workflow consumers; bounds truncation. Bounded by subscriber count. + map consumers = 9; + + StreamLifecycle lifecycle = 10; +} + +message ProducerCursor { + int64 seq = 1; + int64 first_offset = 2; // replayed on a duplicate append + int64 count = 3; + // Distinguishes a genuine retry from a client reusing a sequence with + // different content, which must be rejected rather than deduplicated. + bytes content_hash = 6; + // Set by FinishWriting. Ends this producer's writes without closing + // the stream for anyone else. + bool fenced = 4; +} + +message ConsumerCursor { + string workflow_id = 1; + string run_id = 2; + int64 offset = 3; + // Pins truncation; see 8.4. + bool active = 4; +} + +message StreamLifecycle { + google.protobuf.Duration retention = 1; + int64 max_items = 2; // 0 = unbounded + int64 max_bytes = 3; // 0 = unbounded +} +``` + +Size is O(producers + consumers), not O(items). That is the property that keeps this off the CHASM partial-read critical path. + +### 2.3 Service + +```protobuf +service StreamService { + rpc CreateStream(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_STANDARD; } + rpc AddMessages(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_STANDARD; } + rpc FinishWriting(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_STANDARD; } + rpc PollMessages(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_LONG_POLL; } + rpc DescribeStream(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_STANDARD; } + rpc CloseStream(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_STANDARD; } + rpc TruncateStream(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_STANDARD; } + rpc DeleteStream(...) { business_id = "frontend_request.stream_id"; category = API_CATEGORY_STANDARD; } + rpc ListStreams(...) { category = API_CATEGORY_STANDARD; } // visibility-routed, not shard-routed +} +``` + +Options follow `chasm/lib/activity/proto/v1/service.proto`. `business_id` drives shard routing; `API_CATEGORY_LONG_POLL` puts `PollMessages` in the right quota bucket. `ListStreams` goes through the CHASM visibility manager (`chasm.VisibilityManager.ListExecutions`) rather than shard routing, which means the `Stream` component declares a `Visibility` field and a business-ID alias. + +`FinishWriting` records a per-producer fence and does not close the stream. It orders behind every append that producer already issued, so the claim holds under concurrency. Closing is stream-wide and separate. + +--- + +## 3. Storage mapping + +### 3.1 Branch + +Each stream mints one branch at creation: + +```go +branchToken, err := shard.GetExecutionManager().GetHistoryBranchUtil().NewHistoryBranch( + namespaceID, streamID, runID, + treeID, // the stream's CHASM execution RunID + nil, // branchID: generated + nil, // no ancestors + 0, 0, retention, +) +``` + +The OSS implementation (`common/persistence/history_branch_util.go:49`) ignores namespace, workflow, and run, and returns `{TreeId, BranchId, Ancestors}`. Passing them anyway keeps the SaaS override (Walker) able to do whatever it needs. See §11. + +### 3.1a Offset bucketing + +`history_node` in Cassandra is `PRIMARY KEY ((tree_id), branch_id, node_id, txn_id)`, so the partition key is `tree_id` alone and every node of a tree shares one partition. That is safe for Workflow History because history is capped by `HistorySizeLimitError` and `HistoryCountLimitError`. A stream is deliberately uncapped, so a single tree per stream would grow a Cassandra partition without bound. + +So a stream is not one tree. It is a sequence of trees, one per fixed-size offset bucket: + +``` +bucket = offset / bucket_size +treeID = uuid.NewSHA1(streamNamespaceUUID, []byte(streamID + "/" + bucket)) +branchID = uuid.NewSHA1(streamNamespaceUUID, []byte(streamID + "/" + bucket + "/b")) +``` + +Both are derived arithmetically, so there is no index to store and no index to grow. `bucket_size` is chosen by the server at creation and is immutable for the stream's life, because changing it would renumber existing offsets. + +Consequences: + +- A batch never crosses a bucket boundary. Split at the boundary before staging. +- A read spanning buckets issues one range read per bucket and concatenates. +- Whole-bucket truncation is `DeleteHistoryBranch` on that bucket's token, which reclaims a whole partition instead of leaving per-row tombstones. Partial truncation inside the live bucket just advances `base_offset`. + +**Bucketing does not weaken the chain rule of §3.5**, but the argument is worth stating because it is not obvious. Reads restart the transaction chain per tree, so in principle a stale node could be accepted as a bucket's first node. It cannot, because bucket boundaries are offset-aligned: a stale node is either the bucket's first node, in which case it shares a node ID with the real first node and loses on transaction ID, or it is preceded within its bucket by valid nodes, in which case the chain rule applies exactly as it does within one tree. This needs a test, listed in §15. + +### 3.2 Offset to node ID + +`serializeAppendRawHistoryNodesRequest` rejects `nodeID <= 0` with "eventID cannot be less than 1" (`common/persistence/history_manager.go:429-433`). So: + +``` +nodeID = (offset % bucket_size) + 1 +``` + +API offsets start at 0 and are global across buckets. The node ID is bucket-relative, which is why the `+ 1` and the modulo both matter. This mapping is internal and must never leak into the wire protocol. + +### 3.3 Append + +```go +persistence.AppendRawHistoryNodesRequest{ + ShardID: shardID, + BranchToken: s.BranchToken, + IsNewBranch: s.HeadOffset == 0, + Info: streamInfo(namespaceID, streamID), + History: blob, // serialized StreamMessageBatch, opaque to the store + NodeID: s.HeadOffset + 1, + PrevTransactionID: s.LastTxnID, + TransactionID: txnID, // from the shard's transaction ID generator +} +``` + +Verified opaque: `AppendRawHistoryNodes` (`history_manager.go:501`) passes `request.History` straight through and only reads `len(.Data)` for size accounting. Nothing parses it. + +`transactionSizeLimit()` caps the blob (`history_manager.go:437-442`), so a batch that exceeds it must be split across nodes before the transition commits. + +### 3.4 Read + +```go +persistence.ReadHistoryBranchRequest{ + ShardID: shardID, + BranchToken: s.BranchToken, + MinEventID: fromOffset + 1, + MaxEventID: s.HeadOffset + 1, // exclusive + PageSize: pageSize, + NextPageToken: token, +} +``` + +`ReadRawHistoryBranch` returns `HistoryEventBlobs []*DataBlob`, `NodeIDs []int64`, and a page token, without parsing. + +### 3.5 The clip invariant and the transaction-ID chain + +> **Reads clip to `HeadOffset`, and the store drops any node whose transaction ID went backwards. Together those are sufficient; no prepare phase is required.** + +Node append and frontier update are not one atomic store operation, and never were for workflow history either. Cassandra's `execution_store.go:110-126` appends history nodes in a loop and then calls `UpdateWorkflowExecution`; SQL does the same at `sql/execution.go:339`. + +Three failure shapes: + +1. **Nodes written, frontier not advanced.** Orphan nodes sit at offsets at or past `HeadOffset`. `MaxEventID` excludes them, so no reader observes them. +2. **Duplicate node IDs from a retry.** `filterHistoryNodes` (`history_manager.go:1039-1073`) keeps the highest transaction ID per node ID. +3. **Orphans beyond the retry's extent, later shadowed by the advancing frontier.** This is the case clipping alone does not cover, and it is the one an objection to this approach would reach for. + +Concretely for (3): an append writes nodes at offsets 100 and 110 under transaction `T1` and fails. The producer retries with a smaller batch, writing only node 100 under `T2`, and the frontier advances to 105. A later append writes node 105 under `T3`. Node 110 is now below `HeadOffset` and clipping would expose it. + +The store already handles this. `filterHistoryNodes` requires transaction IDs to be non-decreasing as node IDs increase and skips anything that regresses: + +```go +if node.TransactionID < lastTransactionID { + continue +} +``` + +with the contract stated in its own comment at `:1063-1066`: "event batches with larger node ID -> batch with lower transaction ID is invalid (happens before)". Node 110 carries `T1 < T3`, so it is dropped. This is the same mechanism that protects workflow history after a failed transaction, and it is why `PrevTransactionID` exists on the append request. + +**Requirement this places on us:** transaction IDs must come from the shard's monotonic generator, never from a per-stream counter, and each attempt must take a fresh one. Reusing a transaction ID across attempts breaks the chain rule. + +**Verified.** `TestStreamLog_ShrinkingRetryDropsStaleNode` in `common/persistence/tests/history_store_stream_log.go` builds exactly the layout above and asserts the stale node is absent from both `ReadHistoryBranch` and `ReadRawHistoryBranch`. Passing on SQLite and Postgres. The raw assertion is the load-bearing one: `ReadHistoryBranch` has a contiguity check that independently catches a leak, and a stream never goes through it. + +There is no window in which a reader sees a gap, and no window in which two readers disagree about a prefix. + +### 3.6 Orphan reclamation + +Correctness does not depend on cleanup, but storage does. Streams produce orphans more often than workflow history does, through publish retries and truncate races, so a slow background scavenger is the wrong cadence. + +`TrimHistoryBranch(BranchToken, NodeID, TransactionID)` takes a known-good frontier and removes everything off the valid chain. The `Stream` component holds exactly that frontier in `head_offset` and `last_txn_id`. + +So: on a failed append, schedule a CHASM pure task that calls `TrimHistoryBranch` with the committed frontier. Cheap, targeted, and it runs seconds after the failure rather than hours. Coalesce with `WithSingletonTask` so a burst of failures produces one trim. + +If measurement shows orphan volume outrunning this, a periodic per-shard sweep is the fallback. Instrument orphan bytes from the first benchmark so the question is answered with data. + +--- + +## 4. RPCs + +### 4.1 `AddMessages` + +``` +AddMessages(namespace, stream_id, producer_id?, seq?, expected_offset?, owner_epoch?, messages[]) + -> { first_offset, next_offset, head_offset } +``` + +Handler calls `chasm.UpdateComponent(ctx, ref, (*Stream).AddMessages, req)`. Inside the transition, in this order: + +1. `Closed` -> `FailedPrecondition` with reason `StreamClosed`. +2. **Dedup.** If `producer_id` is set and `producers[producer_id].seq == seq`, compare `content_hash`. Matching content returns the recorded `first_offset` and `count` without appending. **Differing content is rejected** with `InvalidArgument`, because silently returning the old offsets would drop the caller's data and look like success. A stale or skipped sequence is rejected with the last accepted sequence in the error details so the caller can resynchronise. +3. **Write fence.** If `producers[producer_id].fenced` -> `FailedPrecondition` with reason `ProducerFinished`. +4. **Ownership fence.** If `owner_epoch` supplied and below `state.owner_epoch`, return `FailedPrecondition` with reason `ProducerFenced`. +5. **Compare-and-append.** If `expected_offset` supplied and it differs from `head_offset`, return `AlreadyExists` carrying `head_offset` so the caller can resynchronise. +6. Serialize `StreamMessageBatch`, splitting if over `transactionSizeLimit`. +7. Emit pending log appends (§5) at `nodeID = head_offset + 1`, taking a fresh transaction ID from the shard generator (§3.5). +8. `head_offset += len(messages)`; `last_txn_id = txnID`; record the producer cursor. +9. **Inline retention truncation.** If `lifecycle.max_items` or `max_bytes` is set and now exceeded, advance `base_offset` and queue the trim, bounded by the consumer pin (§8.4). Doing this at the end of a successful append avoids a separate sweeper. + +Acknowledge after the transaction commits. `first_offset` is the offset of the first message; the caller derives per-message offsets by position. + +### 4.1a Group commit (designed, deferred) + +**Not in the first cut.** The serialized one-transition-per-append path is easier to reason about and to prove, and the case for group commit rests on Cassandra numbers we are not measuring in this prototype. The design is recorded here so the public offset contract does not have to change when it lands. + +A stream linearizes through one CHASM execution, so its ceiling is the transition rate on that execution. On Cassandra a transition is a lightweight transaction, and per-partition LWT throughput is the binding constraint. + +This design costs **one** transition per append, because there is no prepare phase. Group commit divides even that: concurrent `AddMessages` calls for the same stream that arrive while a transition is in flight are queued, and the next transition applies all of them in order, assigning each a contiguous offset range and emitting one log append per member. + +- Each member keeps its own dedup entry and its own `first_offset`, so idempotency is unaffected. +- A member that fails validation (closed, fenced, offset mismatch) is rejected individually without aborting the group. +- Group size is capped by `stream.maxGroupCommitSize` and by the aggregate blob size against `transactionSizeLimit`. + +Combined with clients batching several items into one call, this is what keeps a hot stream inside the ceiling. The 1-pager forbids user-facing batching knobs, and this respects that: group size is chosen by the server from what happens to be in flight, not configured by the application. + +`producer_id` and `expected_offset` are alternative idempotency mechanisms. `producer_id` suits a retrying activity; `expected_offset` suits a caller that already tracks position. Supplying neither gives at-least-once, which is a valid choice for a caller that does not care. + +### 4.1b Flush + +A producer can append a `STREAM_MESSAGE_KIND_FLUSH` marker to say "the turn is finished, deliver what you have". It carries no payload and consumes one offset like any other message. + +It matters for Path C. Without it, a consumer that has drained the current slice has no way to distinguish "the producer is still generating" from "the producer is done", and the only remaining signal is a timeout. An LLM turn has a natural end, the producer knows when it happens, and making it say so removes a tuning knob rather than adding one. + +The server does not interpret flush beyond delivering it. Whether a flush ends an iteration is the consuming application's decision, which keeps the semantics out of the server and matches leaving topic meaning to the application. + +Note this is a delivery hint, not a lifecycle event. It is weaker than `FinishWriting` (which fences a producer) and much weaker than `CloseStream` (which seals the stream). All three exist because they answer different questions: "this turn is done", "I am done", "the stream is done". + +### 4.2 `PollMessages` + +``` +PollMessages(namespace, stream_id, from_offset, max_items, max_bytes, + topics[], wait_new_messages, wait_timeout) + -> { messages[], first_offset, next_offset, closed, close_reason, head_offset } +``` + +`topics` filters at read time. Filtering is by exact topic match only, no predicates, because a predicate would have to run server-side over payloads the server cannot decode. Offsets are assigned over the unfiltered stream, so `next_offset` always advances past everything read, filtered out or not. That keeps cross-topic ordering available to callers who pass no filter. + +Topic filtering forces the server to decode the batch envelope (not the payloads) to inspect each message's `topic`. That is the second case, alongside §4.3, where a batch is deserialized; both are bounded by batch size. + +1. `from_offset < base_offset` -> `OutOfRange` with reason `Truncated`, carrying `base_offset` so the reader can jump forward rather than fail. +2. `from_offset > head_offset` -> `InvalidArgument`. +3. `from_offset < head_offset`: serve. Tail cache first (§6); on miss, `ReadRawHistoryBranch` across the buckets the range spans. Trim to `max_items` and `max_bytes`. Return. + **If the read returns fewer messages than `[from_offset, head_offset)` implies and the shortfall is at or above `base_offset`, return `DataLoss`, never a short or empty page.** Committed data that has gone missing is the worst failure a durable stream can have, and reporting it as an empty stream lets a consumer conclude the producer simply had nothing to say. The frontier in the component is what makes this checkable without a separate manifest. +4. `from_offset == head_offset` and `closed`: return empty with `closed = true`. +5. `from_offset == head_offset`, not closed, `wait_new_messages`: long-poll (§4.4). +6. Otherwise return empty immediately. + +### 4.3 Reading from mid-batch + +A batch is one node, so `from_offset` can land inside one. Two options: + +- **Chosen:** the server deserializes only the boundary batch and drops the leading messages. One batch, bounded by batch size, and only on the first page. +- Rejected: return whole batches and let the reader skip. That leaks framing into the protocol and makes `max_bytes` unenforceable. + +Every other batch on the page stays opaque and is forwarded as-is. + +### 4.4 Long-poll + +```go +chasm.PollComponent(ctx, ref, func(s *Stream, ctx chasm.Context, from int64) (pollOut, bool, error) { + return pollOut{head: s.HeadOffset, closed: s.Closed}, + s.HeadOffset > from || s.Closed, + nil +}, fromOffset) +``` + +The predicate is monotonic, which `chasm.PollComponent` requires (`chasm/engine.go:426-429`). `HeadOffset` only increases and `Closed` never clears, so it holds. + +`PollComponent` subscribes before releasing the execution lease (`service/history/chasm_engine.go:743-765`), which closes the subscribe/notify race that `get_workflow_util.go:136` has to re-read around. + +Timeout uses `contextutil.WithDeadlineBuffer` with per-namespace `stream.longPollTimeout` and `stream.longPollBuffer`, matching `chasm/lib/activity/handler.go:210-215`. On soft timeout, return an empty response with `next_offset = from_offset`, and the client re-polls. That is the established convention and it keeps a slow stream from looking like an error. + +**Long-poll, not gRPC server-streaming.** Frontend runs 24 unary interceptors and 2 streaming ones (`service/frontend/fx.go:285-328`), and `common/authorization/interceptor.go:224` cannot resolve a namespace at stream handshake because there is no request body to read. Server-streaming is the right eventual read API. It needs new auth, rate-limit, and redirection plumbing, and it is not on the path to proving the cost claim. + +--- + +## 5. The CHASM transaction hook + +This is the one framework change, and the only item with an owner outside this project. + +### 5.1 Problem + +A CHASM component cannot contribute append-log batches at transaction close. `ChasmTree` (`service/history/interfaces/chasm_tree.go:19-53`) has no method for it. Without the hook, an append is two persistence calls: `AppendRawHistoryNodes`, then a separate CHASM update to advance the frontier. + +### 5.2 What already works + +`UpdateWorkflowExecutionRequest.UpdateWorkflowEvents` is `[]*WorkflowEvents`, and each entry carries its own `BranchToken`. Multi-branch appends in a single request are structurally supported. `WorkflowEvents.Events` is typed `[]*historypb.HistoryEvent` though, so it cannot carry an opaque blob. + +### 5.3 Change + +Add a sibling list rather than overloading `WorkflowEvents`: + +```go +// common/persistence/data_interfaces.go +type LogAppend struct { + BranchToken []byte + NodeID int64 + PrevTxnID int64 + TxnID int64 + Blob *commonpb.DataBlob + IsNewBranch bool + Info string +} + +type UpdateWorkflowExecutionRequest struct { + // ... existing fields + UpdateLogAppends []*LogAppend + NewLogAppends []*LogAppend +} +``` + +`executionManagerImpl.UpdateWorkflowExecution` (`common/persistence/execution_manager.go:168`) folds these into the same `[]*InternalAppendHistoryNodesRequest` it already builds from `UpdateWorkflowEvents`, reusing `serializeAppendRawHistoryNodesRequest`. + +On the CHASM side, `CloseTransaction` gains a way to surface pending appends: + +```go +// service/history/interfaces/chasm_tree.go +CloseTransaction() (chasm.NodesMutation, []*persistence.LogAppend, error) +``` + +with `MutableStateImpl.closeTransaction` passing them through. A component registers a pending append during its transition via a new `MutableContext` method. + +### 5.4 Fallback + +If the hook slips, both producer paths still work as two persistence calls in the same order, with the clip invariant unchanged. Cost is one extra round trip per batch, which is still far below a signal. Sequence the hook first but do not block Stage 1 on it. + +--- + +## 6. Tail cache + +A shard-local ring per open stream, holding the last N appended blobs keyed by offset. + +- Populated on append, so the producer's own bytes are already in memory. +- Read path checks it before touching persistence, which makes a reader at the tail a memcopy. +- Bounded per stream and in aggregate per shard, evicting oldest first. A miss falls through to `ReadRawHistoryBranch`, so the cache is never load-bearing for correctness. +- Dropped when the shard loses ownership. A new owner rebuilds it from appends. + +This is what makes fan-out cheap. N readers at the tail cost N memcopies rather than N range scans, which is the difference between the 10-subscriber limit and no meaningful limit. + +`service/history/chasm_notifier.go` uses one global mutex and says so in TODOs at lines 16 and 18. Acceptable for a prototype; a real item before fan-out at scale. + +--- + +## 7. Path A: workflow publishes to its own stream + +An attached stream is a subcomponent of the workflow's execution, so it is already on the workflow's shard and inside the workflow's lock. + +New command handled in `RespondWorkflowTaskCompleted`: + +```protobuf +COMMAND_TYPE_ADD_STREAM_MESSAGES + +message AddStreamMessagesCommandAttributes { + string stream_id = 1; // empty = the workflow's default output stream + repeated StreamMessage messages = 2; +} +``` + +Handling goes in `chasm/lib/workflow/` next to the existing command handlers. The handler resolves the attached `Stream` subcomponent and runs the §4.1 transition against it. The appends ride the workflow task's existing commit via §5. + +Cost of publishing: one extra blob in a write that was already happening. No history event, no extra round trip, and the workflow is not rescheduled. + +Publishing to a stream the workflow does not own is not supported by this command. Use `AddMessages`. See §12. + +--- + +## 8. Path C: workflow consumes a stream + +### 8.1 Mechanism + +A workflow subscribes by recording a `ConsumerCursor`. Thereafter, when the server builds a workflow task for that execution, it attaches the pending slice: + +``` +PollWorkflowTaskQueueResponse.stream_slices: [ + { stream_id, from_offset, to_offset, messages[] } +] +``` + +and records the range it delivered as an attribute on the event that closes the task: + +``` +WorkflowTaskCompletedEventAttributes.stream_cursors: [ + { stream_id, from_offset, to_offset } +] +``` + +The response field carries bytes and serves the live delivery. The event +attribute carries offsets only. Replay is served from the event attribute, not +from the response field, for the reason in §8.3. + +**No new event type.** The range rides an event that already exists once per task, so in-workflow consumption adds zero events to history. + +### 8.1c Subscribing from inside the workflow + +Both stream commands write one history event. Subscribing writes `WorkflowStreamSubscribed`, carrying the stream id and the resolved start offset. Publishing writes `WorkflowStreamMessagesAdded`, carrying the stream id and the offset range the batch landed at. + +The reason is not cost. Every SDK matches issued commands against command-generated events **in order**, popping a queue as each event arrives (`workflow_machines.rs`, `self.commands.pop_front()`). A command that produces no event leaves its entry at the head of that queue and the next event pops the wrong one. So a command reachable from workflow code has to have an event, and the codebase already says so: `TestBufferEvent` exists to force exactly that, and this design had been opting out of it. + +Subscribing costs one event per subscription, which is the same order as a single signal. Publishing costs one per batch, which is the case that needed measuring rather than assuming, because unlike a subscription it recurs. + +It also closes an operational gap. Without the events nothing in History explains why a workflow began receiving stream data, or that it published any. + +Consumption is still zero events per task: the offsets a task consumed ride `WorkflowTaskCompleted`, and payloads never enter History on any path. + +##### What a per-batch event costs + +Measured by `TestStreamPublishHistoryCost`, which runs a workflow whose single Workflow Task carries N publish commands and then completes, and differences against an N=0 run so the fixed opening and closing events cancel. + +| what was published | per batch | per message | +|---|---|---| +| 100 batches x 1 message x 20 bytes | 41 bytes | 41 bytes | +| 100 batches x 1 message x 2000 bytes | 40 bytes | 40 bytes | +| 100 batches x 10 messages x 20 bytes | 41 bytes | 4.1 bytes | +| the same 100 messages as Signals, 20 bytes | 112 bytes | 112 bytes | +| the same 100 messages as Signals, 2000 bytes | 2097 bytes | 2097 bytes | + +Three things follow. + +The event is a fixed 41 bytes. A hundredfold increase in payload size does not move it, which is the claim that bodies never enter History, stated as a measurement rather than a reading of the code. + +Batching is free. The event is per call, so a batch of a thousand costs History what a batch of one costs. Against Signals the saving is 2.7x at 20-byte messages and 51x at 2KB, and it grows without bound with payload size. + +The binding limit is the event count, not the size. At 41 bytes, 51,200 events is the `limit.historyCount.error` ceiling while using 2MB of the 50MB size budget. So a workflow is bounded at 51,200 publish **calls**, with no bound on the messages inside them. Today's Signal path is bounded at 10,000 signals **and** carries every payload against the size limit, so a 2KB-per-message workflow exhausts history at roughly 25,000 messages. The stream path reaches 51,200 batches of any size, which at 100 messages per batch is 5.12M messages for 2MB of History. + +The guidance that follows is to batch, and that is what `workflow.add_stream_messages` documents. A workflow that would genuinely exceed 51,200 calls continues as new, which the cursors already survive (§8.6). + +#### Resolution + + + +`COMMAND_TYPE_SUBSCRIBE_STREAM` carries only a stream id and a start offset. Everything else the cursor needs is resolved by the server, because a workflow cannot look it up: a stream's collection id is its run id and its bucket size is its own, and reading either means I/O. A value the workflow carried would be a reading rather than a fact, so it could differ on replay. + +Where the resolution happens is forced by two constraints. The command handler runs under the state lock with nowhere to do I/O from, and by delivery time the cursor has to already exist. So a subscription to a stream in another execution is staged by the handler and resolved in the flush before commit, the same place staged log writes go. A stream the workflow owns needs no resolution and is subscribed in the handler directly. + +A negative start offset resolves to the stream's frontier once, at registration, and the resolved value is recorded. Left to delivery it would be resolved again on replay against a stream that has since moved. + +### 8.1a Where the cursor lives, and how a range becomes a fact + +The cursor is a subcomponent of the **consuming workflow**, not of the stream. + +That placement is what makes the advance atomic. A workflow's CHASM nodes and its History events travel in the same `WorkflowMutation` (`UpsertChasmNodes` alongside the events, `common/persistence/data_interfaces.go:367`), so folding a delivered range into the cursor lands in the same transaction as the `WorkflowTaskCompleted` event that records it. Held on the stream instead, every advance would be a cross-execution write, and a crash between the two writes would either redeliver a range or skip one with nothing in History to show it. + +This also settles a question left open in §5: **the CHASM transaction hook is not required for Path C.** It remains wanted for the producer side, where the log append is a genuinely separate persistence write, but consumption needs nothing new. + +A range therefore becomes a fact in two steps: + +1. **At delivery**, the range attached to the task is staged on the cursor as a pending range. Staging never advances the cursor. +2. **At completion**, the pending range is recorded on the event and folded into the cursor, in one transaction. + +A task that fails or times out recorded nothing, so its staged range never became history. The next delivery re-reads from the unchanged cursor and simply overwrites what was staged, which is why redelivery is allowed to produce a different range than the attempt before it. + +The stream keeps a separate `ConsumerCursor` as a **truncation floor** only. It is advisory for retention and is not the position anything is served from, so it can lag without affecting correctness. + +### 8.1b What the prototype implements + +Delivery, staging, recording and the cursor advance are built and covered by `tests/stream_consume_test.go`. Two limits are worth naming rather than leaving to be discovered: + +**Consuming a stream in another execution is built except for its last hop.** A consumer cannot read another execution's frontier while closing its own transaction, so the frontier is pushed to it instead: an append that leaves a registered external consumer behind schedules a side-effect task on the stream, which writes the new frontier onto that consumer's cursor. That write dirties the consuming execution, and its own transaction close then sees the cursor is behind and schedules a workflow task, which is the same path an owned stream takes. Delivery clips to the pushed frontier rather than a live read, so no cross-execution read happens while a workflow transaction is open. + +The subscription registers the truncation pin on the stream **before** writing the cursor on the workflow. Interrupted after the first write there is a pin holding storage nothing reads, which costs space; the other order would leave a cursor with no pin, and truncation would be free to take a range it still points at. + +A stream's log is read from the **stream's own shard**, not the consumer's. History nodes are stored per shard, so an external stream read from the consumer's shard returns no nodes at all. That surfaces as `Workflow execution history not found` from the workflow task start, which matching logs as a task it could not start and drops, so the symptom is a workflow that never receives its task rather than an error anyone sees. The shard is derived from the stream id through `Config.GetShardID`, the same mapping the rest of the server uses. + +One consequence worth stating: an external consumer's pin never advances, because advancing it would be a cross-execution write on the workflow task path. A stream with a live external consumer therefore does not truncate below the offset that consumer subscribed at. A capped slice relies on the same pushed frontier to continue, so it resumes on the next push rather than immediately. + +**Replay reassembly is built.** When a workflow task carries History, every `WorkflowTaskCompleted` in it that recorded a range gets its payloads re-read from the log and attached, tagged with that event's id. A response therefore holds at most one untagged slice, for the task about to run, plus one per recorded range being replayed. Python reads both, through the sdk-core work in §8.1c, so the consuming end is proven end to end. + +The cost is the one §8.3 flagged: a delivery carrying full History re-reads every range that History records. Sticky delivery carries only the tail and pays proportionally less, but a cold replay of a long-lived consumer re-reads everything it ever consumed, and that still has no bound. + +One implementation detail with a cost attached: both the delivery and completion paths resolve the workflow component **read-only first**, and only take the mutable path when a cursor exists. Reaching it mutably marks the node dirty, which would add a node to the transaction of every workflow in the cluster, subscribed or not. That showed up as a task-generation change in `TestRefreshSubStateMachineTasks` before the read-only check was added. + +### 8.2 What must be recorded, and why empty counts + +Two rules, both load-bearing: + +**Record on every task where the stream is subscribed, including when `from_offset == to_offset`.** A task in which the subscription observed nothing is a fact replay must reproduce, not an absence of a fact. If empty ranges are omitted, replay has no record that the workflow reached that point with nothing available, so it is free to deliver items the workflow did not have then. The resulting divergence does not surface at the point of the error; it surfaces later as an unrelated nondeterminism failure, which makes it expensive to diagnose. + +Riding `WorkflowTaskCompleted` is what makes this affordable. A separate event per task per subscribed stream would have made the idle case cost an event. + +**The first record for a subscription carries the resolved start offset.** "Subscribe from the current tail" resolves against `head_offset` at subscribe time, which is a nondeterministic reading. Recording the resolved value turns it into a fact. This applies whether or not anything was delivered on that task. + +### 8.3 Determinism, and where the bytes come from on replay + +On replay the server reads `[from_offset, to_offset)` from the same branch and attaches the same bytes. That read is deterministic because: + +- the log is immutable, so a given offset always holds the same bytes; +- the range is recorded, so it does not depend on when replay happens; +- `filterHistoryNodes` resolves the node chain the same way on every read; +- every task carries a record, so the sequence of observations is fully reconstructible. + +What that leaves open is the carrier: which channel hands those bytes to the worker when the task being replayed is not the current one. The response field cannot do it. It is built once per delivery (`CreateRecordWorkflowTaskStartedResponseWithRawHistory`, `service/history/api/recordworkflowtaskstarted/api.go:400`), so it holds a single slice set, while a cache miss replays every prior task. + +The Go SDK already answers this for Updates, and the answer is a constraint on us rather than a choice. In `ProcessWorkflowTask` the message index is built from one of two sources depending on the path (`internal/internal_task_handlers.go:1106`): + +- live: `indexMessagesByEventID(taskMessages)`, from the poll response; +- replay: `indexMessagesByEventID(historyMessages)`, synthesized from the event stream by `inferMessageFromAcceptedEvent` while iterating events (`:518`). + +Two consequences: + +1. **On replay the SDK takes per-event data from History only.** It never calls back to the server inside the replay loop, and nothing in that loop does I/O. So "the SDK re-reads each recorded range during replay" is not a small change. It puts a blocking network call inside the deterministic replay path. +2. **A missing body is already a hard failure rather than a nondeterminism error.** `if historyMessages[i].Body == nil { return nil, fmt.Errorf("missing body in message for update ID %v", ...) }` (`:1120`). That is the shape §8.4 asks for when stream data has expired, and it already exists. + +Delivery position is solved too: messages are drained by event ID, before and after each `ProcessEvent` (`:1203`, `:1221`), so a slice keyed to the `WorkflowTaskCompleted` event ID lands at the same point live and on replay. + +**Decision: the server reassembles slices on the History read path, keyed by the recorded cursor.** Storage keeps the property the whole design rests on, because only offsets are written to History. Reassembly happens on read: the bytes are fetched from the stream branch and attached to the event that recorded the range. This matches the carrier the SDK already expects, so it needs no change to the replay loop. + +The Update precedent buys the same guarantee by writing the payload into the event (`accepted_request`, whose proto comment says it exists "so that the worker can recreate and deliver that same message as part of replay"). We keep the recreate-from-History contract and move the cost from write to read. + +That cost is real and belongs on the read path's budget: one stream read per subscribed workflow per cache miss, on a path shared with `GetWorkflowExecutionHistory` and the UI. §6's tail cache covers the common case, where replay follows soon after the writes. Cold replay of an old workflow is the expensive case and needs a bound before this is more than a prototype. + +**Segmentation is not a concern here.** One slice arrives per workflow task, the SDK drains it once, and replay drains an identical slice once. A client-side design that reads a backend continuously while a task is open has to reconstruct how many times the consumer was woken within the task, because condition evaluation happens per wake. Putting the delivery boundary at the task boundary, which is already a durable boundary, removes that problem rather than solving it. + +### 8.4 Truncation interlock + +Replay needs the bytes to still exist. So truncation is bounded: + +``` +effective_base = min(requested_base, min over consumers of consumer.offset) +``` + +`TruncateStream` below a registered consumer cursor is rejected with `FailedPrecondition`. A consumer is deregistered when its workflow closes, which releases the floor. + +This is the one place where a consumer constrains the stream, and it is unavoidable: recording a cursor instead of the data means the data has to outlive the cursor. + +Implemented for attached streams: subscribing registers a pin on the stream in the same transaction that creates the cursor, and committing a delivered range advances the pin with it. `Truncate` rejects a base past the lowest active pin, and the message cap yields to it rather than dropping a range a consumer recorded a cursor for. Registering and advancing in the cursor's own transaction is what keeps the two from drifting: a pin lost while its cursor survived would leave truncation free to take a range the cursor still points at. + +The interlock covers deliberate truncation. It cannot cover retention expiry on a stream whose consumer outlives it, or out-of-band deletion. If replay finds a recorded range below `base_offset`, the workflow task **fails retryably** with a distinct error rather than raising a nondeterminism error. The distinction matters operationally: a nondeterminism error looks like a code bug and gets triaged as one, while "the stream data this workflow needs is gone" is an infrastructure condition with a different fix. An operator can restore or extend retention and the workflow proceeds. + +### 8.4a The alternative: holding the task open + +Option 7 in the canonical options doc solves the same problem by keeping the workflow task open and reading an external store directly inside it, completing when the stream is idle or a flush arrives. It is cheaper, because no data enters Temporal, and it needs no server work. + +The structural difference is where the delivery boundary sits. Ours is the workflow task boundary, which is already durable and already recorded, so replay reproduces it for free. Option 7's boundary is wherever the reader happened to be when the task ended, so it has to be reconstructed: stream-empty markers, and per its own prototype's ADR-018, the number of drains as well, because `wait_condition` evaluates once per activation. + +The cost of our choice is a workflow task per slice instead of one long task. The benefit is bounded task duration, which keeps us clear of the workflow-task timeout question (gap K3, still open) and avoids pinning a worker slot for the length of an LLM response. `design-comparison.md` §4 has the full comparison. + +### 8.5 Bounded attachment and the wake exception + +The attached slice is capped by **both** `stream.maxConsumeBytesPerTask` and `stream.maxConsumeItemsPerTask`. Bytes alone is not enough: a burst of many tiny messages stays under a byte cap while producing a slice large enough to make one task's drain unboundedly long. Whichever limit binds first, attach a prefix, record only that range, and schedule a follow-up workflow task. + +That is the single intentional exception to "publishing never wakes a workflow". + +Implemented in `closeTransactionHandleWorkflowTaskScheduling` rather than at workflow task completion. Completion is the obvious place and it is not enough: registering a subscription against a stream that already holds data completes no workflow task of its own, so a completion-time check leaves that workflow waiting for unrelated traffic. Transaction close covers both, and the pending-task check runs first so a workflow that already owes a task never pays for the subscription lookup. Publishing does not. An **active in-workflow subscription** does, because the workflow asked to be woken. Worth stating explicitly, because it is the property that keeps Path C from silently reintroducing the cost Path B removes. A workflow that does not subscribe is never woken by stream traffic. + +### 8.6 Continue-as-new and reset + +- **Continue-as-new**: the cursor is workflow state, carried to the successor by the server rather than by the application. The stream is untouched, and because offsets are global to the stream the successor resumes at exactly the offset its predecessor stopped at, so nothing is duplicated or dropped. Without this the subscription would end silently: the stream keeps its consumer pin and keeps pushing to the workflow id, and the successor would have nowhere to put what arrives. + + Only subscriptions to streams in other executions are carried. A stream the workflow owns lives in its execution and does not itself survive the run transition yet (§8a.1), so a cursor for one would leave the successor pointing at a stream it cannot reach. +- **Reset**: the workflow rewinds; the stream does not. Cursor events before the reset point are intact, so replay works, and the new run re-consumes from the cursor as of that point. Relative to the abandoned run, some messages are delivered twice. That is visible to the application by design, matching the decision that rewinds are the application's concern rather than something the system hides. + +--- + +## 8a. Run transitions for an attached stream + +A standalone stream is unaffected by anything the workflow does. An attached stream lives in the workflow's execution, so a run transition moves it, and a reader addressed at the old run has to be able to follow. + +### 8a.1 Continue-as-new, retry, cron + +Successor creation already persists the old-run mutation and the new-run snapshot as one update. The stream rides that: + +1. Apply any `AddStreamMessages` commands before the closing command. A stream command after a closing command is invalid. +2. Copy the bounded state into an equivalent child under the successor root, preserving stream ID, first execution run ID, `base_offset`, `head_offset`, `last_txn_id`, `bucket_size`, and the producer map. +3. Set `redirect_run_id` on the old child to the successor's run ID. +4. Persist old mutation, new snapshot, staged log appends, and the current-run pointer together. + +Continue-as-new does not close the stream, and an append in the same workflow task as the continue-as-new is carried at its post-append head. + +A poll holding a reference to the old run sees `redirect_run_id` set, follows it, and keeps its offset. Offsets are global across runs, so nothing about the cursor changes. The old child stays as a redirect target through retention, which is what stops an in-flight long poll from stalling on a superseded run. + +### 8a.2 Reset + +Reset rebuilds workflow state from an earlier point. It must not rewind the stream, because consumers may already have read past that point, and offsets never decrease. + +A stream command's history event names an offset range but carries none of the stream's state, and its `Apply` is a no-op, so replay still cannot reconstruct that state from History. Reset therefore carries it forward explicitly: + +1. Take the current execution's lock through the existing reset path. +2. Rebuild the target mutable state from history. +3. Read the current run's stream children under that lock. +4. Replace any replay-derived stream children in the reset snapshot with copies of the **current** state, not the state as of the reset point. +5. Mark the replaced run's streams redirected to the reset run. +6. Commit through the existing conflict-resolution request. + +An append racing a reset conflicts on the current run's database condition, so one of them retries. If the append commits first, the reset copies the advanced head. If the reset commits first, the append follows the redirect and lands on the reset run. There is no interleaving in which the reset becomes current with a head older than an acknowledged append. + +Work repeated after a reset appends new messages at new offsets. The server does not try to detect that they are semantically the same as earlier ones; the producer metadata is there so the application can. + +--- + +## 9. Lifecycle + +| Operation | Mechanism | +|---|---| +| Identity, standalone | caller-supplied `stream_id`, unique in the namespace | +| Identity, attached | `(namespace, workflow_id, first_execution_run_id, stream_name)`. The first execution run ID is stable across a continue-as-new chain and prevents collisions after workflow ID reuse | +| Create, standalone | `CreateStream`, or implicitly on first `AddMessages` when the caller opts in | +| Create, attached | Materialized by the server when a stream reference is passed to `StartWorkflow` | +| Producer done | `FinishWriting` sets a per-producer fence; the stream stays open for others | +| Close | `CloseStream`, or a pure task fired when the owning execution completes. The owner link is on the **business ID**, not the run ID, so it survives continue-as-new | +| Truncate, explicit | `TrimHistoryBranch` plus advancing `base_offset`, bounded by §8.4 | +| Truncate, cap-driven | `max_items` / `max_bytes` evaluated inline at the end of a successful append (§4.1 step 9). No sweeper: the append transition is already writing, so folding the check into it costs nothing and keeps the cap tight | +| Retention | Side-effect task at `close_time + retention`, then `DeleteHistoryBranch` and `chasm.DeleteExecution` | +| Continue-as-new | Standalone: nothing to do. Attached: copy state to the successor and redirect the old child (§8a.1) | +| Reset | Carry the current head forward, do not rewind (§8a.2) | + +Close seals, it does not delete. A closed stream stays readable through retention, which is what removes the shutdown handshake that Workflow Streams needs today. + +Archival is out of scope, matching non-workflow CHASM executions today, which take a delete path rather than an archival path. When generic CHASM archival lands, streams pick it up. + +--- + +## 10. Failure analysis + +| Scenario | Behaviour | +|---|---| +| Crash between node append and frontier advance | Orphan nodes at or past `head_offset`, invisible to readers. Retry rewrites the same node IDs; the store keeps the highest transaction ID. Trim task reclaims the space (§3.6). | +| Retry writes a **smaller** batch than the failed attempt | Trailing orphan nodes end up below the advancing frontier. The transaction-ID chain rule in `filterHistoryNodes` drops them (§3.5). This is the case clipping alone does not cover. | +| Producer retries after a timeout it did not observe | Dedup on `(producer_id, seq)` returns the original `first_offset`. No double append. The producer never has to reason about whether the append landed. | +| Two producers race | `expected_offset` mismatch returns `AlreadyExists` with the current head. With `owner_epoch`, the stale producer is fenced outright. | +| Producer publishes after `FinishWriting` | `FailedPrecondition` with reason `ProducerFinished`. Other producers are unaffected. | +| Shard failover mid-stream | New owner reloads the component. `owner_epoch` bump fences the old producer. Readers resume from their own offset. The tail cache is rebuilt. | +| Reader disconnects mid-page | No server state to clean up. The reader re-polls from its last offset. | +| Reader polls a truncated range | `OutOfRange` carrying `base_offset`, so the reader can jump forward. | +| Close races an in-flight append | Both serialize through the component transition. Either the append lands before close, or it fails `FailedPrecondition`. | +| Workflow consuming a stream that gets truncated | Prevented by §8.4. | +| Workflow replays a range lost to retention expiry | Retryable workflow task failure with a distinct error, not a nondeterminism error (§8.4). | +| Batch exceeds `transactionSizeLimit` | Split across nodes inside the transition, before commit. | +| One member of a group commit fails validation | Rejected individually; the rest of the group commits (§4.1a). | +| Subscribed workflow's task carries no new messages | An empty range is still recorded, so replay reproduces the observation (§8.2). | +| Retried sequence carries different content | Rejected with `InvalidArgument`. Never deduplicated into a silent data drop (§4.1). | +| Committed node missing under the frontier | `DataLoss`, never a short page (§4.2). | +| Poll addressed at a run that continued as new | Follows `redirect_run_id` and keeps its offset (§8a.1). | +| Append races a reset | Conflicts on the current-run condition; one side retries and follows the redirect (§8a.2). | +| Stream outgrows one Cassandra partition | Cannot happen; offsets roll to a new tree every `bucket_size` (§3.1a). | + +--- + +## 11. Cross-boundary check: Walker + +OSS `NewHistoryBranch` ignores `namespaceID`, `workflowID`, and `runID`, but the `HistoryBranchUtil` interface accepts them because the SaaS storage layer overrides it. Minting branches whose tree ID is not a workflow run ID is a change to an assumption Walker may rely on. + +This needs a read of `saas-temporal/walker/` and a conversation with that team before this goes beyond prototype. It does not block an OSS prototype, and it is listed here so it is not discovered late. + +--- + +## 12. Explicitly out of scope + +- **Cross-shard publish to a stream the producer does not own, atomic with the producer's own state transition.** Producers use `AddMessages`, which is idempotent by offset and reliable. Two-phase commit or an outbox only becomes necessary if atomicity with the producer's transition is required, and token streaming does not require it. +- **Nexus and cross-namespace.** A layer on top. +- **gRPC server-streaming reads.** See §4.4. +- **A non-durable tier.** Reintroduces the tuning knob the design removes. +- **Server-populated rewind metadata.** The field exists; filling it in is a later opt-in. +- **Cassandra.** See §16. +- **A pluggable external backend.** Moving payloads to Redis or similar is a real product option and a working prototype of it exists (`design-comparison.md` §4), but it puts durability outside Temporal, which is the thing this design exists to avoid. The read API here is offset plus long-poll, the same shape an external adapter exposes, so the two could later sit behind one client API without either being rewritten. +- **Push-based in-workflow delivery.** Delivering items to a consuming workflow as signals would put per-item cost back into the consumer's history and hand dedup to user code. Pull-at-task-build (§8) avoids both. + +--- + +## 13. Configuration + +| Key | Default | Purpose | +|---|---|---| +| `stream.enabled` | false | Per-namespace kill switch | +| `stream.longPollTimeout` | 20s | Matches history long-poll convention | +| `stream.longPollBuffer` | 3s | Deadline buffer | +| `stream.maxBatchBytes` | 2MB | Bounded by `transactionSizeLimit` | +| `stream.bucketSize` | 100000 | Messages per history-node tree (§3.1a). Immutable per stream once created; changing the default affects new streams only | +| `stream.maxMessagesPerPoll` | 1000 | Read page bound | +| `stream.maxBytesPerPoll` | 4MB | Read page bound | +| `stream.tailCacheBytesPerStream` | 1MB | Fan-out cache | +| `stream.tailCacheBytesPerShard` | 256MB | Aggregate bound | +| `stream.maxConsumeBytesPerTask` | 1MB | Path C attachment bound | +| `stream.maxConsumeItemsPerTask` | 1000 | Path C attachment bound; binds where messages are tiny | +| `stream.deliverOnFlush` | true | Cut a Path C slice at a flush marker rather than only at the size bounds | +| `stream.maxGroupCommitSize` | 16 | Appends coalesced into one transition (§4.1a) | +| `stream.maxSubscribersPerStream` | 0 | 0 = unbounded; present as a safety valve | + +No user-facing batching knob. Batching is the client's choice of how many messages to put in one `AddMessages` call, and server-side coalescing is internal. Removing the tuning knobs is a stated goal of the 1-pager, and it is why `maxGroupCommitSize` is an operator dial rather than an API field. + +## 14. Metrics + +The claims in the high-level design are unmeasured, so the prototype has to emit what settles them: + +| Metric | Why | +|---|---| +| persistence round trips per append | the core cost claim | +| conditional writes per append, and realised group size | tests whether group commit does what §4.1a says | +| orphan bytes outstanding, and trim task latency | tests the objection in `design-comparison.md` §5 | +| append to reader-receipt latency, p50 and p99 | the 100ms batching bar | +| tail-cache hit rate | tests the fan-out claim | +| stream count, bytes, and items per namespace | capacity planning and, later, pricing | +| long-poll wakes per delivered message | the CHASM notifier is execution-scoped, so an attached stream's pollers wake on unrelated workflow changes. Wake amplification is a real risk and needs measuring, not assuming | + +**Stream and topic names must never be metric tags.** They are user-supplied and unbounded, so tagging on them is a cardinality incident waiting to happen. + +--- + +## 15. Test plan + +**Unit** (`chasm/chasmtest`, in-memory engine): +- Offset assignment across batches, including split batches. +- Dedup by `(producer_id, seq)` returns the original offsets. +- `expected_offset` mismatch returns the current head. +- Epoch fencing rejects a stale producer. +- `FinishWriting` fences one producer and leaves others writing. +- Close rejects subsequent appends. +- Truncation floor respects registered consumers. +- Cap-driven truncation fires inline on append and respects the consumer pin. +- Mid-batch read trims correctly at every boundary. +- Topic filter returns the right subset and still advances `next_offset` past filtered-out messages. +- Group commit: N concurrent appends produce contiguous non-overlapping ranges, one transition, and per-member dedup entries; one invalid member does not abort the group. + +**Storage-level** (against the real `history_node` store, `common/persistence/tests`): +- **The shrinking-retry case from §3.5.** Write nodes at 100 and 110 under `T1`, then node 100 alone under `T2`, advance the frontier to 105, write node 105 under `T3`, and assert a read of `[100, 120)` never returns the node at 110. This is the single most important test in the suite: it is the case that decides whether reusing `history_node` is sound, and it is the objection an external reviewer will raise first. +- `TrimHistoryBranch` with the committed frontier reclaims orphans and leaves the valid chain intact. +- **Bucket-boundary staleness.** The §3.5 shrinking-retry layout, arranged so the abandoned attempt straddles a bucket boundary and the stale node is the first node of the next bucket. Asserts the chain rule still rejects it once the frontier moves past. This is the test behind the claim in §3.1a that bucketing is safe, and it is the one most likely to surprise us. + +**Functional** (`tests/stream_test.go`, against SQLite and Postgres): +- Produce and consume end to end, single and many subscribers. +- Long-poll wakes on append and returns empty on soft timeout. +- Reader below `base_offset` gets `OutOfRange` with a usable floor. +- Stream stays readable after the owning workflow closes. +- Continue-as-new: a poll in flight across the boundary follows the redirect and returns a contiguous offset sequence with no gap and no repeat. +- Reset: the reset run inherits the current head, and an append racing the reset ends up on exactly one of them. +- Paths A and C using `tests/testcore/taskpoller.go:29`, whose `WorkflowTaskHandler func(task) ([]*commandpb.Command, error)` lets a test emit `AddStreamMessages` and read the attached slice directly. **No SDK fork is needed to prove either path.** + +**Durability:** +- Kill the server mid-append, restart, assert readers see a prefix and never a gap. +- Assert a producer retry after the kill does not double-append. + +**Failover:** +- Force shard movement mid-stream, assert the epoch fence rejects the stale producer and readers resume. + +**Replay (Path C):** +- Run to completion, evict the workflow cache, force replay, assert attached slices are byte-identical. +- **Idle-task replay.** A workflow subscribed to a stream that receives nothing for several tasks must replay identically. Assert an empty range was recorded per task, and that replay delivers nothing on those tasks. Without the §8.2 rule this test fails, so it is the guard on that fix. +- **Resolved start offset.** Subscribe from the tail while the stream already has items, complete, replay, and assert the workflow sees the same starting position rather than re-reading from zero. +- **Attachment bound.** A backlog larger than both `maxConsumeItemsPerTask` and `maxConsumeBytesPerTask` is split across tasks, and replay reproduces the same split. +- **Retention loss.** Delete the branch data underneath a workflow that recorded a range, force replay, and assert a retryable task failure rather than a nondeterminism error. + +--- + +## 16. Benchmark methodology + +Same workload both ways, on SQLite and Postgres. + +Workload: one LLM-shaped producer emitting 40 messages per second of roughly 20 bytes each for 60 seconds, with 1, 5, and 25 concurrent subscribers. + +Baseline: today's Workflow Streams pattern (batched Signals plus a polling Update), reproduced as a functional-test workload. + +Report: +- persistence round trips per message, +- history bytes written per message, +- Actions per message, +- p50 and p99 time from append to reader receipt, +- marginal cost of the Nth subscriber, +- whether 100ms batching holds up, which is the bar the 1-pager sets. + +The benchmark is the deliverable that makes the September 14 decision possible. Everything else is in service of it. + +--- + +## 17. Sequencing + +| Stage | Content | Notes | +|---|---|---| +| 0 | Baseline harness and measurements | Nothing to compare against without it | +| 0b | **Storage-level spike** | **Done.** `common/persistence/tests/history_store_stream_log.go`, passing on SQLite and Postgres, with a negative control. Blob opacity and the §3.5 shrinking-retry case both hold | +| 1 | Component, log helpers, unit tests | | +| 1b | CHASM transaction hook (§5) | Raise with the CHASM owner in week one; has a fallback | +| 2 | RPC surface and wiring | `service/frontend/service.go:507`, `service/frontend/fx.go`, `common/api/metadata.go`, `service/frontend/configs/quotas.go` | +| 3 | Long-poll, tail cache, group commit | Where the fan-out and throughput claims get proven | +| 4 | Lifecycle: close, write fence, truncate, retention, owner binding, `ListStreams` | | +| 5 | Path A, workflow publish | Needs the api-go additions | +| 6 | Path C, workflow consume | Highest risk, sequenced last | +| 7 | Benchmark, demo, write-up | | + +**API dependency.** Stages 5 and 6 need `go.temporal.io/api` changes: `temporal.api.stream.v1`, `COMMAND_TYPE_ADD_STREAM_MESSAGES`, `COMMAND_TYPE_SUBSCRIBE_STREAM`, a `stream_slices` field on `PollWorkflowTaskQueueResponse`, and a `stream_cursors` field on `WorkflowTaskCompletedEventAttributes`. Consumption adds no event type, because the consumed range rides an event that already exists (§8.1). Both commands do need one, `WorkflowStreamSubscribed` and `WorkflowStreamMessagesAdded`, for the command-to-event matching in §8.1c. Plan is one branch on `temporalio/api` pinned by pseudo-version rather than a local `replace`, so the branch stays buildable by others. `make update-go-api` is the existing path. + +--- + +## 18. What this design does not answer + +- Whether per-row cost on Cassandra changes the batching decision. Out of scope here, and the SQLite and Postgres numbers must not be read as answering it. +- Pricing. The design makes cost track bytes rather than item count, which is the shape the 1-pager wants, but the model is not settled. +- Whether the collection primitive Metablock needs should be built on this or the reverse. This design does not need it, which is a scheduling argument rather than an architectural one. +- Naming. +- The UI story: reconstructing a stream in the Web UI, and dropping the Signal and Update clutter. +- Whether orphan volume under real retry rates stays inside what the eager trim reclaims (§3.6). Instrumented from Stage 0, not answered by design. +- What bounds the cost of cold replay under the §8.3 decision. Reassembling slices on the History read path is cheap while the tail cache is warm and unbounded when it is not. A very old workflow with a long consumed history is the case that needs a limit, and it does not have one yet. +- Whether `GetWorkflowExecutionHistory` should reassemble at all, or only the worker-facing read. The UI and `tctl` share that path, so reassembly there means stream payloads appear in operator tooling that today only sees offsets. That is arguably desirable for debugging and arguably a size and redaction problem. Not decided. +- Cross-cluster replication. History-node data already replicates, so the mechanism is inherited rather than designed, but the conflict semantics for a stream written on two sides of a failover are not worked out. Last-writer-wins is the assumed answer and it is lossy. +- Whether the 51,200-call publish ceiling needs anything beyond continue-as-new. Batching moves it far out of reach for the workloads this targets, but a workflow that publishes one message per call in a tight loop hits it, and nothing warns before it does. A per-batch minimum, or coalescing repeated calls within one task, would both change the command-to-event matching, so neither is free. diff --git a/streaming-high-level-design.md b/streaming-high-level-design.md new file mode 100644 index 00000000000..8782bb58a4a --- /dev/null +++ b/streaming-high-level-design.md @@ -0,0 +1,309 @@ +# Native Streams: High-Level Design + +| | | +|---|---| +| Status | Draft for review | +| Ticket | [AI-198](https://temporalio.atlassian.net/browse/AI-198), epic [AI-37](https://temporalio.atlassian.net/browse/AI-37) | +| Project | D1, Native streaming (Win the Agent Loop) | +| Author | Moe Dashti | +| Date | 2026-08-23 | +| Companion | `streaming-detailed-design.md`, `design-comparison.md`, `streaming-benchmark-results.md` | + +This is a clean-room design, derived from Temporal's storage invariants. It was written without reading the earlier prototypes. Those have since been compared against it in `design-comparison.md`, and the changes that comparison produced are folded in here. + +--- + +## 1. Problem + +Interactive agents produce a continuous stream of tokens, tool calls, reasoning traces, and progress updates. Users need to see them as they happen. + +Today the answer is **Workflow Streams** (Public Preview): an Activity batches output into **Signals**, and an app server long-polls the workflow with a `poll_events` **Update**. The developer-facing shape is right. The mechanics are not: + +- Batching intervals sit at seconds, not milliseconds, to amortise per-item overhead. +- Items land in the workflow's Event History, so they count against the 50MB cap, are re-read on every replay, and are duplicated or dropped across continue-as-new. +- `MaximumSignalsPerExecution` defaults to 10000 (`common/dynamicconfig/constants.go:2630`). A token-per-signal stream exhausts that inside one long response. +- At most 10 concurrent subscribers, which is `history.maxInFlightUpdates`. Measured, it does not degrade gracefully: 25 subscribers deliver 44% of receipts with a 40s p99. +- A second ceiling that has not been discussed: `history.maxTotalUpdates` is 2000 **per workflow execution**, and every poll spends one. At 100ms batching, five subscribers exhaust a workflow's lifetime read budget in under a minute, forcing continue-as-new for reasons that have nothing to do with the application. +- The stream is unreadable once the workflow closes, so producer and consumer have to coordinate a shutdown. +- Cost. Customers describe it as a non-starter, and several run Redis alongside Temporal instead. + +The last point is the commercial one. Streaming is cited in lost and degraded accounts (Replit, Adobe, Dust, Harvey), and competitors market our lack of a stream primitive as a differentiator. + +## 2. First principles + +### 2.1 What Temporal is, mechanically + +Strip away the programming model and Temporal is a sharded, single-writer, fenced state store. Every entity hashes to one history shard, that shard is owned by one history host at a time under `range_id` fencing, and all mutations linearize through that owner. That single serialization point is the whole source of Temporal's consistency. + +Behind that writer sit **two storage shapes**, and they have opposite cost curves: + +| | Mutable state | History nodes | +|---|---|---| +| Access pattern | read-modify-write | append-only | +| Addressing | by entity | by offset, within a branch | +| Cost of a change | proportional to **total size** | proportional to the **delta** | +| Read granularity | whole blob, loaded eagerly | paged range `[min, max)` | +| Practical size bound | yes | none | +| Already has | fork, conditional update | fork, trim, delete, range read, replication | + +Temporal's own Workflow implementation exploits this asymmetry: the events go in the log, and only the summary goes in mutable state. + +### 2.2 What a stream is, mechanically + +An ordered sequence of immutable opaque blobs, written by one logical producer at a time, read concurrently by many readers each holding their own cursor, terminated by a close marker. + +Read that against the table above: + +- A stream is **never** read-modify-write. Nothing mutates an item after it is written. +- A stream is **unbounded**. That is the requirement, not an accident. +- A stream is **not a decision input**. The producer's next action does not depend on any consumer. + +A stream is exactly the append-only log shape, and exactly not the mutable-state shape. + +### 2.3 Where the cost comes from today + +Workflow Streams have no choice but to use the wrong shape, because the only append-only log a user can write to is workflow history, and every append to it schedules a workflow task. Per batch, that buys: + +- a `WorkflowExecutionSignaled` event plus a mutable-state update, +- a `WorkflowTaskScheduled` event plus a transfer-task row, +- a full worker round trip (matching dispatch, `RecordWorkflowTaskStarted` re-taking the workflow lock, replay, `RespondWorkflowTaskCompleted` as another two-write transaction), +- and then a polling Update to get the data back out, which is another state transition. + +None of that is buying durability. It is buying *workflow semantics* for data that has none. + +### 2.4 The three separations + +Each separation removes exactly one cost centre, and each rests on something Temporal already relies on. + +**1. The stream gets its own log, not the workflow's.** + +The history-node store is already documented as decoupled from workflow concepts (`common/persistence/data_interfaces.go:1158`: "V2 regards history events growing as a tree, decoupled from workflow concepts"). `NewHistoryBranch` (`common/persistence/history_branch_util.go:49`) ignores namespace, workflow, and run entirely; a branch token is just `{TreeId, BranchId, Ancestors}`. + +So give each stream its own branch. That alone delivers: no history bloat, no 50MB cap, no continue-as-new entanglement, readable after the workflow closes, unbounded size, and independent retention. + +**2. Appending does not schedule a workflow task.** + +A token is data *produced by* an execution, not a decision input *to* it. Nothing in the workflow's state machine advances because a token arrived. Scheduling a workflow task for it is a category error. + +Removing it removes the transfer-task row and the worker round trip, which is the dominant per-batch cost. + +**3. Reading is not a state transition.** + +The reader's position is the reader's state. If the reader supplies its offset, the server keeps no durable per-subscriber record. Subscriber count stops being a durable-state problem and becomes a memory problem, so the limit of 10 goes away and a poll costs no Actions. + +## 3. The model + +### 3.1 Entity + +A **Stream** is a first-class entity addressed by `(namespace, streamId)`, implemented as a CHASM component. It exists in two arrangements: + +- **Standalone**: its own CHASM execution, routed by `streamId`. Independent of any workflow. +- **Attached**: a subcomponent of a workflow's execution, so it is co-located on that workflow's shard. + +Both are readable and writable by clients over the same API. The difference is only which shard owns it and, therefore, whether the owning workflow can publish to it for free. + +### 3.2 What lives where + +Only the **frontier** needs to be linearized, so only the frontier lives in mutable state: + +``` +Stream (CHASM component; size is O(1) regardless of stream length) + BranchToken its own history-node branch + HeadOffset visibility frontier; readers never see at or past this + BaseOffset truncation floor + LastTxnID node chaining + Closed, CloseReason + OwnerEpoch producer fence + ProducerCursors per-producer dedup and write fence, bounded by producer count + Consumers registered in-workflow cursors, bounded + Owner optional execution ref, for lifecycle only + Lifecycle retention, item and byte caps +``` + +Payload bytes go to the stream's own branch and never enter the CHASM tree. + +A stream is not one branch but a sequence of them, one per fixed-size offset bucket. `history_node` partitions on `tree_id` alone, which is safe for Workflow History because history is capped and unsafe for a stream because it is not. Rolling to a new tree every `bucket_size` offsets keeps any one Cassandra partition bounded, and because the bucket is `offset / bucket_size` and the tree ID is derived from it, there is still no index to store. + +That last sentence is the design's main claim. It means the component does not grow with the stream, there is no segment-index to blow up mutable state, and **there is no dependency on CHASM partial reads** (`OSS-4917` and `OSS-4918`, both still `To Do`). The history-node store already does paged range reads; that is its job. + +### 3.3 Guarantees + +- **Total order** within a stream, by offset. +- **Exactly-once write**: an acknowledged append appears exactly once, under producer retries, shard failover, and concurrent producers. +- **Durable before acknowledged.** No fire-and-forget tier. Durability is the reason to be on Temporal at all. +- **Readers see a prefix.** A reader never sees a gap and never sees an item that a later reader will not see. +- **Missing data is reported as loss, never as emptiness.** If committed data is gone, a read returns `DataLoss` rather than a short page. Silently reporting an empty stream is the worst failure a durable stream can have, because the consumer concludes the producer had nothing to say. +- **At-least-once delivery to the reader, made exactly-once by the reader's cursor.** The reader owns its offset, so a duplicate poll is idempotent. In-workflow consumption is exactly-once outright, because the delivered range is recorded (§4.3). +- **The server never sees plaintext.** Items are opaque blobs on the write path, in storage, and on the read path. The payload codec runs entirely in the SDK. This falls out of using the raw append and range-read paths, which never deserialize. +- **Multiple topics per stream**, filtered at subscribe time, so cross-topic ordering is preserved for callers who want it. +- **A producer can finish without closing the stream.** A write fence lets one producer declare it is done while others keep publishing. Closing is a separate, stream-wide act. + +### 3.4 Why exactly-once needs no new commit protocol + +The write pair here is (append log nodes, then conditionally advance the frontier). That is what workflow history has done for a decade, and two mechanisms already in the store make it safe. + +**First, readers clip to `HeadOffset`.** A crash between the two steps leaves orphan nodes at offsets at or past `HeadOffset`, which no reader can observe. + +**Second, the store enforces a transaction-ID chain.** `filterHistoryNodes` (`common/persistence/history_manager.go:1039-1073`) does more than prefer the higher transaction ID when two rows share a node ID. It requires transaction IDs to be non-decreasing as node IDs increase, and discards any node whose transaction ID went backwards. Its own comment states the rule: "event batches with larger node ID -> batch with lower transaction ID is invalid (happens before)". + +The second mechanism is what covers the case the first does not. Suppose an append writes two nodes, at offsets 100 and 110, then fails. A retry writes only one node at 100, and the frontier advances past it. A later append writes at 105. The stale node at 110 is now *below* the frontier, so clipping alone would expose it. The chain rule drops it, because its transaction ID precedes the one at 105. + +With both in place, exactly-once reduces to two single-field checks inside the conditional update that is already happening: + +- **Producer fence.** `OwnerEpoch`, bumped on ownership change. A stale producer's update fails. +- **Idempotency.** The producer supplies `(producerId, seq)` or an `expectedOffset`. A retry returns the original result instead of appending again. + +This is the design's second claim: getting the storage shape right removes the need for a prepare phase, a two-phase commit, and the machinery that goes with them. + +Orphan rows still occupy space until reclaimed. `TrimHistoryBranch` exists for exactly this, takes a known-good frontier as its input, and the Stream component holds that frontier, so the sweep is a cheap task fired after a failed append rather than a slow background scavenger. This matters because streams generate orphans more often than workflow history does, through publish retries and truncate races. + +## 4. Access paths + +### 4.1 Path B: off-shard producer + +The common case. LLM tokens come from an Activity calling the model, not from workflow code. + +``` +Activity / client ──AddMessages(streamId, items)──▶ stream's shard + │ + append blob to stream branch + advance HeadOffset + │ +App server ◀──PollMessages(from=N, wait)─────────┘ + ──SSE──▶ browser +``` + +One gRPC hop, one log append, one small conditional update. The workflow is not involved at all: no workflow task, no history event, no worker round trip, no lock on the workflow. + +### 4.2 Path A: the workflow publishes to its own stream + +For application-level progress updates ("planning", "calling search", "writing file"), which Johann's own analysis argues matter more than token streaming once models get fast. + +An attached stream is on the workflow's shard, under the workflow's lock, inside the workflow task's existing commit. A new `AddStreamMessages` command appends to the stream's branch as part of that transaction. + +The marginal cost of publishing is one extra blob in a write that was already happening. Zero history events, zero extra round trips. + +### 4.3 Path C: the workflow consumes a stream + +The 2026-07-23 sync recorded no proposed solution for this and the Bellevue session deferred it. That has since changed: **Option 7** in the [canonical options doc](https://app.notion.com/p/3b28fc567738812f8c67ca3ebdf9ce38), added from the 2026-08-14 call, targets bucket 2 by holding a workflow task open and reading an external store directly. So what follows is an alternative to Option 7, not the only answer on the table. `design-comparison.md` §4 puts the two side by side. + +**Record the cursor in history, not the data.** + +The server attaches the pending slice `[cursor, HeadOffset)` to the workflow task response, and records the range it delivered on the `WorkflowTaskCompleted` event that closes that task: + +``` +WorkflowTaskCompletedEventAttributes.stream_slices: [ + { streamId, fromOffset, toOffset } +] +``` + +On replay the server re-reads the same offset range from the same branch. That is deterministic by construction, because the log is immutable and offset-addressed. Nothing about the replay depends on timing. + +Two details make it actually correct rather than nearly correct: + +- **The range is recorded on every task where the stream is subscribed, including when it is empty.** A task in which the subscription observed nothing is a fact replay has to reproduce, not an absence of a fact. Recording only non-empty deliveries would let replay hand the workflow items it did not have at that point, and the divergence would surface much later as an unrelated nondeterminism error. Riding the existing `WorkflowTaskCompleted` event rather than writing a separate event is what makes the empty case free. +- **The first record for a subscription carries the resolved start offset**, because "start from the current tail" is otherwise a nondeterministic decision made at subscribe time. + +History then grows with **workflow tasks, not with items**, and it adds no events at all. A 50,000-token response consumed across 8 workflow tasks costs 8 small attribute entries instead of 50,000 events. + +This is the third claim, and it inverts the framing from the July discussion. That discussion looked for ways to make many small workflow tasks cheaper (pipelining). The cheaper move is to make one workflow task carry many items. + +Two properties fall out of putting the boundary at the workflow task: + +- **Segmentation is not a problem.** One slice arrives per task, the SDK drains it once, and replay drains an identical slice once. There is no need to reconstruct how many times the consumer was woken within a task, which is the hardest part of doing this from the client side. +- **Publishing still never wakes a workflow, but an active subscription does.** If a subscribed workflow's cursor is behind `HeadOffset` when its task completes, the server schedules another task. That is the workflow asking to be woken, which is a different thing from a producer waking it. A workflow that does not subscribe is never woken by stream traffic. + +## 5. Lifecycle + +- **Creation** is explicit for standalone streams, and implicit for attached ones (pass a stream reference to `StartWorkflow` and the server materialises it), so producers and consumers never have to coordinate on who creates it. +- **Identity.** An attached stream is keyed on `(namespace, workflowId, firstExecutionRunId, streamName)`. The first execution run ID keeps the identity stable across a continue-as-new chain while preventing collisions after workflow ID reuse. +- **Write fence.** A producer can declare it has finished writing without closing the stream. The fence orders behind everything that producer published before it, so the claim holds even with concurrent callers. +- **Close** is explicit, and automatic when the owning execution completes. Close seals the stream; it does not delete it. The owner link is on the business ID, so it survives continue-as-new. +- **Retention** works like a workflow's. A closed stream stays readable through retention, then `DeleteHistoryBranch` reclaims it. +- **Truncate** advances `BaseOffset` and calls `TrimHistoryBranch`. A reader below `BaseOffset` gets a distinguishable error carrying `BaseOffset`, so it can jump forward rather than fail. Cap-driven truncation is evaluated inline at the end of a successful append rather than by a background sweeper, and it is pinned by any registered in-workflow consumer's cursor. +- **Continue-as-new.** A standalone stream needs no handling. An **attached** stream lives in a run that is about to be superseded, so its bounded state is copied to the successor and the old child is marked as redirecting to the new run. A long poll in flight follows the redirect and keeps its offset, since offsets are global across runs. +- **Reset** rewinds the workflow but never the stream, because consumers may already have read past the reset point. The reset run inherits the *current* head, not the head as of the reset point. An append racing a reset conflicts on the current-run condition, so exactly one of them wins and the other retries. + +## 6. Positioning and the throughput ceiling + +The target is **many modest-volume streams**, not few enormous ones: thousands to millions of concurrent streams per namespace, tens to thousands of items each, up to a few hundred items per second on a hot one. For cross-business aggregation the right tool is Kafka, and this design does not try to be that. + +That positioning is not a marketing choice, it follows from the architecture. A stream linearizes through one CHASM execution, so its throughput ceiling is the transition rate on a single execution. On Cassandra that is a lightweight-transaction rate per partition, conservatively budgeted at around 100 per second. + +Two things keep us under it: + +- **One transition per append, not two.** Because there is no prepare phase, an append costs one conditional update plus one non-conditional log write. A design that needs prepare-then-commit pays two. +- **Group commit.** Concurrent appends to the same stream coalesce into a single transition, dividing the per-append transition cost by the group size. Combined with client-side batching of items into one call, this is what puts the target comfortably inside the ceiling. + +A stream that consistently needs a large group to keep up is a signal to split traffic across streams. That is the application's call, not a server tuning knob. + +## 7. Cost + +Per 100ms batch, steady state. The middle column is what we measure in Stage 0, not an estimate. + +| | Workflow Streams today | This design (Path B) | +|---|---|---| +| gRPC hops | 2, plus a worker round trip | 1 | +| History events written | 2 per signal, plus WFT completion | 0 | +| Log appends | 1, into the workflow's own history | 1, into the stream's branch | +| Mutable-state updates | 2 or more | 1, small and fixed-size | +| Transfer-task rows | 1 | 0 | +| Worker round trips | 1 | 0 | +| Workflow lock acquisitions | 3 or more | 0 | +| Counts against 50MB history | yes | no | +| Hard ceiling | 10000 signals per execution | per-stream transition rate (§6) | +| Cost of the Nth subscriber | an Update per poll | a memcopy | +| Conditional writes per batch | 2 or more | 1, divided by the group-commit size | + +**Both columns are now measured**; see `streaming-benchmark-results.md`. Over the same workload at 100ms batching: + +| | Workflow Streams | Native | +|---|---|---| +| persistence ops per message, 1 subscriber | 1.52 | **0.50** | +| persistence ops per message, 25 subscribers | 11.30 | **0.55** | +| workflow history bytes per message | 379 to 706 | **0** | +| delivered, 25 subscribers | 19% | **100%** | +| p99, 25 subscribers | 11.2s | **104ms** | + +The number that matters most is not the ratio but the shape: the native cost per message barely moves between 1 and 25 subscribers, because reads cost no persistence operations at all. The current pattern's rises sevenfold over the same range. + +## 8. Non-goals + +- **Sub-100ms realtime voice.** Different point in the design space; durable-per-item is the wrong trade there. +- **Kafka-scale firehose.** Many modest-volume streams, not few enormous ones. +- **Nexus and cross-namespace.** A layer on top, deliberately later. +- **A non-durable tier for token deltas.** Splitting durable application events from ephemeral deltas reintroduces exactly the tuning knob we are trying to remove. If the cost work lands, the split is unnecessary. +- **Automatic rewind handling.** On workflow retry or reset the stream keeps appending; the rewind surfaces as item metadata. Hiding it from the user would be worse than exposing it. +- **Cross-shard atomic publish** to a stream the producer does not own. Producers use the RPC, which is already idempotent by offset. Two-phase commit only becomes necessary if the publish must be atomic with the *producer's own* state transition, and token streaming does not need that. +- **A pluggable external backend.** This is Option 7, and it is a legitimate product option with a working prototype behind it. It is not this design, because it moves durability outside Temporal, which the Native Streams 1-pager currently rules out as a principle. The two are not exclusive though: our read contract is append plus read-from-offset, the same contract Option 7's pluggable store expects, so a Temporal-native stream could sit behind Option 7's SDK model as one provider among several. `design-comparison.md` §4 argues that sequence is better than a fork. + +## 9. Dependencies and risks + +**An event-sourcing boundary extension, needing sign-off.** Stream control state lives in CHASM and is not reconstructible from public Workflow History. That is deliberate and it is what keeps items out of history, but it is an exception to Temporal's event-sourcing model rather than an application of it: public history stays sufficient to reconstruct everything workflow code can observe, while stream control is auxiliary state that survives alongside it. This needs explicit History and CHASM owner approval and a `docs/architecture` note, not just a design review. + +**The one framework change.** `ChasmTree` (`service/history/interfaces/chasm_tree.go:19-53`) gives a component no way to contribute append-log batches at transaction close. `UpdateWorkflowExecutionRequest.UpdateWorkflowEvents` is already `[]*WorkflowEvents`, each carrying its own `BranchToken`, so multi-branch appends in one transaction are structurally supported; the tree just cannot reach them. This hook is what makes Paths A and B single-round-trip. It has an owner outside this project (Yichao, CHASM) and should be raised in week one. If it slips, both paths still work as two persistence calls with the same invariant and one extra round trip. + +**Blob framing and orphan rejection: verified.** Both properties the storage choice rests on now have running tests (`common/persistence/tests/history_store_stream_log.go`), passing on SQLite and Postgres: + +- an arbitrary non-proto blob round-trips byte-identical on a branch whose tree ID is not a run ID; +- a stale node left by a shrinking retry is dropped once the frontier moves past it, on both the parsing and the raw read path. + +The second was checked with a negative control: giving the stale node a higher transaction ID makes the test fail, so it is not passing vacuously. One detail that changes nothing but is worth knowing: the contiguity check that catches the bad case lives on the parsing path, which a stream does not use, so for streams the transaction-ID chain is the only defence. It holds, and the raw-path assertion covers it. + +**Walker.** OSS `NewHistoryBranch` ignores namespace, workflow, and run, but the interface accepts them because the SaaS storage layer uses them. Minting branches that are not tied to a run needs a check against `saas-temporal/walker/` before this goes past prototype. + +**Notifier scaling.** `service/history/chasm_notifier.go` uses a single global mutex and carries TODOs to that effect. Fine for a prototype, real work for fan-out at scale. + +**Cassandra.** Deliberately out of scope for the prototype, and it is the open question gating the batching decision for production. SQLite and Postgres numbers must not be read as answering it. The per-stream ceiling in §6 is a budgeted figure carried over from Johann's analysis, not something we have measured. + +**Orphan volume.** The eager trim assumes failed appends are occasional. If real retry rates under load produce orphans faster than the trim reclaims them, a periodic sweeper comes back. Worth instrumenting from the first benchmark rather than discovering later. + +**This is a design, not working code.** Both prototypes it is compared against run today, one with a verified TLA+ spec and one with a conformance suite and around 40 recorded failure modes. Every cost claim here is unmeasured until Stage 7. + +## 10. Open questions + +- Offset as an integer or an opaque token. Integers are better ergonomics; tokens leave room to change the addressing later. Note that server-assigned dense offsets sidestep a problem an external store has: a reader parked at the tail can always name its position, because the next offset is simply `HeadOffset`. Redis IDs cannot be named before they are written, which is why Max's design needs a cursor to be a position boundary rather than a record identity. +- Whether per-item metadata (workflow, run, original run, attempt) is populated by the server, and whether it is on by default. The rewind model depends on it existing; whether we fill it in is separable. +- Pricing. Data transferred, storage, and active minutes are the candidates. This design deliberately makes cost track bytes rather than item count, which is the shape the 1-pager asks for. +- Naming. "Stream" collides with Kafka Streams. Using `stream` for now. +- Whether the collection primitive that Metablock needs should be built on this, or the other way round. This design does not need it, which is a scheduling argument, not an architectural one. diff --git a/streaming-open-question-pin-ordering.md b/streaming-open-question-pin-ordering.md new file mode 100644 index 00000000000..d5882510a7b --- /dev/null +++ b/streaming-open-question-pin-ordering.md @@ -0,0 +1,65 @@ +# Registering a consumer pin from inside a Workflow Task + +Status: resolved by removing the pin. Kept because the reasoning matters more +than the answer, and because the answer is a weakening of a guarantee. + +## What the question was + +A `SubscribeStream` command cannot resolve anything itself: a command handler +runs under the state lock with nowhere to do I/O from. So the command staged, +and the flush registered the consumer's pin on the stream and then wrote the +cursor onto the workflow, in that order, inside the transaction committing the +workflow task. + +That order was the guarantee. Interrupted between the two there is a pin +holding storage nothing reads, which costs space and is reclaimable. The other +order would leave a cursor with no pin, and truncation would be free to take a +range that cursor still points at. + +A synchronous cross-shard call cannot live there. The workflow's transaction is +open, it holds the execution lock, and the stream may be on another host. Move +the pin out and the ordering goes with it. + +## Why it is no longer a question + +The pin did not do what it claimed. Nothing released it when a consumer +finished, so a stream with a cap kept everything for as long as a consumer had +ever been registered. The floor was not protecting a reader; it was disabling +the cap. An outside review found that, and it is what makes the trade obvious. + +So the pin is gone. Registering a consumer says who to wake when the frontier +moves, and nothing more. Truncation applies the cap it was asked for. + +A consumer that falls behind the floor is told. On the polling path that was +already true: a read below the base returns an error naming where the stream +now starts. The delivery path now does the same, and used not to, which was the +real hazard: it would have read whatever survived and handed the workflow a +range with a hole in it that nothing could see. + +This is what a log with a retention window does everywhere else. A reader that +cannot keep up gets an out-of-range error rather than silent loss, and the +window is the thing the operator asked for. + +## What this costs + +A workflow consuming a capped stream can now be outrun and fail. Before, it +could not be outrun, because the cap did not work. The failure is explicit, it +names the offsets, and it happens when the workflow next tries to read. + +For a stream with no cap and no retention, nothing changes: there is no floor +to fall behind. + +## What it buys, beyond the cap working + +The cross-execution write disappears from the workflow-task transaction, which +is what made this a design question rather than plumbing. Subscribing from +workflow code no longer has to reach another shard while holding the execution +lock, so the last of the three cross-execution steps stops being a problem +rather than being solved with machinery. + +## What is still not settled + +Registering a consumer still happens in the flush, and still reaches the +stream's shard. That is now a call whose failure costs a delayed wake-up rather +than a lost pin, so it can be retried or deferred without risking data. Making +it asynchronous is a tidy-up, not a correctness fix, and is not done. diff --git a/tests/api_fork_test.go b/tests/api_fork_test.go new file mode 100644 index 00000000000..6428272bf1b --- /dev/null +++ b/tests/api_fork_test.go @@ -0,0 +1,57 @@ +package tests + +import ( + "testing" + + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + streampb "go.temporal.io/api/stream/v1" + "go.temporal.io/api/workflowservice/v1" + "google.golang.org/protobuf/proto" +) + +// The api fork has to round-trip the new shapes over the wire, not merely +// compile. A field added without its descriptor would compile and silently +// drop on marshal. +func TestApiForkCarriesStreamShapes(t *testing.T) { + require.Equal(t, enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, enumspb.CommandType(19)) + + cmd := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + StreamId: "s1", + Messages: []*streampb.StreamMessage{{Topic: "tokens"}}, + }, + }, + } + b, err := proto.Marshal(cmd) + require.NoError(t, err) + var back commandpb.Command + require.NoError(t, proto.Unmarshal(b, &back)) + require.Equal(t, "s1", back.GetAddStreamMessagesCommandAttributes().GetStreamId()) + require.Equal(t, "tokens", back.GetAddStreamMessagesCommandAttributes().GetMessages()[0].GetTopic()) + + resp := &workflowservice.PollWorkflowTaskQueueResponse{ + StreamSlices: []*streampb.StreamSlice{{StreamId: "s1", FromOffset: 4, ToOffset: 7}}, + } + rb, err := proto.Marshal(resp) + require.NoError(t, err) + var rback workflowservice.PollWorkflowTaskQueueResponse + require.NoError(t, proto.Unmarshal(rb, &rback)) + require.Equal(t, int64(7), rback.GetStreamSlices()[0].GetToOffset()) + + attrs := &historypb.WorkflowTaskCompletedEventAttributes{ + StreamCursors: []*streampb.StreamCursor{{StreamId: "s1", FromOffset: 4, ToOffset: 4}}, + } + ab, err := proto.Marshal(attrs) + require.NoError(t, err) + var aback historypb.WorkflowTaskCompletedEventAttributes + require.NoError(t, proto.Unmarshal(ab, &aback)) + // An empty range has to survive the round trip: it is the fact that a + // subscription observed nothing, which replay must reproduce. + require.Len(t, aback.GetStreamCursors(), 1) + require.Equal(t, aback.GetStreamCursors()[0].GetFromOffset(), aback.GetStreamCursors()[0].GetToOffset()) +} diff --git a/tests/sdk_server_host_test.go b/tests/sdk_server_host_test.go new file mode 100644 index 00000000000..1bfb2291c05 --- /dev/null +++ b/tests/sdk_server_host_test.go @@ -0,0 +1,66 @@ +package tests + +import ( + "context" + "encoding/json" + "errors" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/tests/testcore" +) + +func TestSDKValidationServerHost(t *testing.T) { + readyPath := os.Getenv("AI198_SDK_SERVER_READY_FILE") + stopPath := os.Getenv("AI198_SDK_SERVER_STOP_FILE") + if readyPath == "" || stopPath == "" { + t.Skip("SDK validation host needs explicit ready and shutdown paths") + } + _, err := os.Stat(stopPath) + require.ErrorIs(t, err, os.ErrNotExist, "a stale shutdown file must not silently stop this host") + env := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err = env.RegisterNamespace(ctx, namespace.Name("default"), 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + require.NoError(t, err) + config := env.GetTestClusterConfig() + metadata := map[string]any{ + "status": "running", "pid": os.Getpid(), "target": env.FrontendGRPCAddress(), + "namespace": "default", "persistence": "SQLite", "history_hosts": config.HistoryConfig.NumHistoryHosts, + "history_shards": config.HistoryConfig.NumHistoryShards, "started_utc": time.Now().UTC().Format(time.RFC3339), + "shutdown_file": stopPath, "source_provenance": "../server-source-provenance.json", + "configuration": "tests/testcore dedicated test-cluster defaults; namespace default added for SDK clients", + } + writeMetadata := func() { + data, marshalErr := json.MarshalIndent(metadata, "", " ") + require.NoError(t, marshalErr) + require.NoError(t, os.WriteFile(readyPath, append(data, '\n'), 0o644)) + } + writeMetadata() + t.Logf("SDK validation server ready at %s in namespace default; shutdown file %s", env.FrontendGRPCAddress(), stopPath) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + deadline := time.NewTimer(2 * time.Hour) + defer deadline.Stop() + for { + select { + case <-ticker.C: + if _, statErr := os.Stat(stopPath); statErr == nil { + metadata["status"] = "stopping" + metadata["stopped_utc"] = time.Now().UTC().Format(time.RFC3339) + writeMetadata() + return + } else if !errors.Is(statErr, os.ErrNotExist) { + require.NoError(t, statErr) + } + case <-deadline.C: + metadata["status"] = "maximum-runtime-reached" + writeMetadata() + return + } + } +} diff --git a/tests/stream_consume_test.go b/tests/stream_consume_test.go new file mode 100644 index 00000000000..8848f973991 --- /dev/null +++ b/tests/stream_consume_test.go @@ -0,0 +1,977 @@ +package tests + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + streampb "go.temporal.io/api/stream/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + streamlib "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" +) + +// Path C: a workflow consuming a stream. The slice rides the workflow task and +// only the offsets it covered are written to History, so consumption costs no +// event of its own no matter how many messages it carried. +func TestStreamWorkflowConsumesWithoutHistoryPayloads(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-consume-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + we, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + // What each task was handed, in the order the tasks ran. + var delivered [][]*streampb.StreamSlice + task := 0 + + //nolint:staticcheck // SA1019: only the deprecated poller can emit this command type. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(resp *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + delivered = append(delivered, resp.GetStreamSlices()) + task++ + if task > 1 { + return nil, nil + } + // First task publishes; nothing is subscribed yet, so it consumes + // nothing. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("first-token")}, Topic: "tokens"}, + {Body: &commonpb.Payload{Data: []byte("second-token")}, Topic: "tokens"}, + {Body: &commonpb.Payload{Data: []byte("third-token")}, Topic: "tokens"}, + }, + }, + }, + }}, nil + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + require.Empty(t, delivered[0], "nothing is subscribed on the first task") + + // Subscribe from the start of the stream. + sub, err := s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, + WorkflowId: id, + StreamName: chasmworkflow.DefaultStreamName, + StartOffset: 0, + }, + }) + require.NoError(t, err) + require.Equal(t, int64(0), sub.GetFrontendResponse().GetStartOffset()) + + // Resubscribing must not rewind. Ranges below the cursor are already in + // History, and moving back would replay them as if they were new. + sub2, err := s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, WorkflowId: id, + StreamName: chasmworkflow.DefaultStreamName, StartOffset: 2, + }, + }) + require.NoError(t, err) + require.Equal(t, int64(0), sub2.GetFrontendResponse().GetStartOffset(), + "a second subscribe must report the cursor already registered") + + // Second task: the published range should arrive on the task itself. + signalWorkflow(t, s, id, we.GetRunId()) + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + slice := currentSlice(t, delivered[1]) + require.Equal(t, int64(0), slice.GetFromOffset()) + require.Equal(t, int64(3), slice.GetToOffset()) + require.Len(t, slice.GetMessages(), 3) + require.Equal(t, "first-token", string(slice.GetMessages()[0].GetBody().GetData())) + require.Equal(t, "third-token", string(slice.GetMessages()[2].GetBody().GetData())) + + // Third task: the cursor is caught up, so the range is empty. An empty + // range is still delivered and still recorded, because a task that saw + // nothing is a fact replay has to reproduce. + signalWorkflow(t, s, id, we.GetRunId()) + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + idleSlice := currentSlice(t, delivered[2]) + require.Equal(t, int64(3), idleSlice.GetFromOffset(), "a caught-up subscription is still attached") + require.Equal(t, int64(3), idleSlice.GetToOffset()) + require.Empty(t, idleSlice.GetMessages()) + + events := env.GetHistory(s.ns, &commonpb.WorkflowExecution{WorkflowId: id, RunId: we.GetRunId()}) + recorded := recordedCursors(events) + + // One record per completed task from the subscription onward, including the + // one that consumed nothing. + require.Len(t, recorded, 2) + require.Equal(t, int64(0), recorded[0].GetFromOffset()) + require.Equal(t, int64(3), recorded[0].GetToOffset()) + require.Equal(t, int64(3), recorded[1].GetFromOffset()) + require.Equal(t, int64(3), recorded[1].GetToOffset(), + "the idle task must record the empty range rather than omit it") + + // The point of the design: offsets are in History, payloads are not. + for _, e := range events { + require.NotContains(t, e.String(), "first-token", + "a consumed payload must never reach History, found in %v", e.GetEventType()) + require.NotContains(t, e.String(), "third-token", + "a consumed payload must never reach History, found in %v", e.GetEventType()) + } +} + +func recordedCursors(events []*historypb.HistoryEvent) []*streampb.StreamCursor { + var out []*streampb.StreamCursor + for _, e := range events { + attrs := e.GetWorkflowTaskCompletedEventAttributes() + out = append(out, attrs.GetStreamCursors()...) + } + return out +} + +func signalWorkflow(t *testing.T, s *streamTestEnv, workflowID, runID string) { + t.Helper() + _, err := s.env.FrontendClient().SignalWorkflowExecution(s.ctx(), &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: s.ns, + WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + SignalName: "wake", + Identity: "tester", + RequestId: uuid.NewString(), + }) + require.NoError(t, err) +} + +// Publishing never wakes a workflow, but an active subscription does. Without +// this the rest of a stream arrives only when something unrelated happens to +// schedule a task. +func TestStreamSubscriptionSchedulesItsOwnWorkflowTask(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-wake-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + publish := func(bodies ...string) []*commandpb.Command { + messages := make([]*streampb.StreamMessage, len(bodies)) + for i, b := range bodies { + messages[i] = &streampb.StreamMessage{Body: &commonpb.Payload{Data: []byte(b)}, Topic: "tokens"} + } + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{Messages: messages}, + }, + }} + } + + var delivered [][]*streampb.StreamSlice + task := 0 + + //nolint:staticcheck // SA1019: only the deprecated poller can emit this command type. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(resp *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + delivered = append(delivered, resp.GetStreamSlices()) + task++ + switch task { + case 1: + return publish("warmup"), nil + case 2: + // Published while subscribed and caught up, so completing this + // task leaves the cursor behind head. + return publish("alpha", "beta"), nil + default: + return nil, nil + } + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + // Subscribe at the tail, so nothing is owed yet. + sub, err := s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, WorkflowId: id, + StreamName: chasmworkflow.DefaultStreamName, StartOffset: -1, + }, + }) + require.NoError(t, err) + require.Equal(t, int64(1), sub.GetFrontendResponse().GetStartOffset()) + + signalWorkflow(t, s, id, "") + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + require.Equal(t, int64(1), currentSlice(t, delivered[1]).GetToOffset(), "nothing new at the tail yet") + + // No signal this time. The subscription owes two offsets, so completing the + // previous task has to have scheduled this one. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + require.Len(t, delivered, 3) + woken := currentSlice(t, delivered[2]) + require.Equal(t, int64(1), woken.GetFromOffset()) + require.Equal(t, int64(3), woken.GetToOffset()) + require.Len(t, woken.GetMessages(), 2) + require.Equal(t, "alpha", string(woken.GetMessages()[0].GetBody().GetData())) + + // And it has to stop. A wake condition that stayed true would spin the + // workflow on empty tasks forever. + pollCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + idle, err := env.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + }) + if err == nil { + require.Empty(t, idle.GetTaskToken(), + "a caught-up subscription must not keep scheduling tasks") + } +} + +// Subscribing to a stream that already has data is the other case where the +// workflow is owed a task it did not ask for by any other means. +func TestSubscribingToABacklogSchedulesAWorkflowTask(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-backlog-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + var delivered [][]*streampb.StreamSlice + task := 0 + + //nolint:staticcheck // SA1019: only the deprecated poller can emit this command type. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(resp *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + delivered = append(delivered, resp.GetStreamSlices()) + task++ + if task > 1 { + return nil, nil + } + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("backlog-1")}, Topic: "tokens"}, + {Body: &commonpb.Payload{Data: []byte("backlog-2")}, Topic: "tokens"}, + }, + }, + }, + }}, nil + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + _, err = s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, WorkflowId: id, + StreamName: chasmworkflow.DefaultStreamName, StartOffset: 0, + }, + }) + require.NoError(t, err) + + // No signal. Subscribing behind the head is itself a reason to run. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + require.Len(t, delivered, 2) + backlog := currentSlice(t, delivered[1]) + require.Equal(t, int64(0), backlog.GetFromOffset()) + require.Equal(t, int64(2), backlog.GetToOffset()) +} + +// History records the offsets a task consumed and never the payloads, so a +// worker replaying that task has to be handed the bytes again. The response +// field alone cannot do it: it is built once per delivery while a cache miss +// replays every prior task, so each re-supplied range travels with the id of +// the event that recorded it. +func TestReplayGetsTheConsumedRangesBackFromTheStream(t *testing.T) { + // Dedicated, because forcing the replay path means evicting the cached + // workflow context, and CloseShard is not allowed on a shared cluster. + env := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-replay-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + we, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + var delivered [][]*streampb.StreamSlice + task := 0 + + //nolint:staticcheck // SA1019: only the deprecated poller can emit this command type. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(resp *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + delivered = append(delivered, resp.GetStreamSlices()) + task++ + if task > 1 { + return nil, nil + } + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("replay-me-1")}, Topic: "tokens"}, + {Body: &commonpb.Payload{Data: []byte("replay-me-2")}, Topic: "tokens"}, + }, + }, + }, + }}, nil + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + _, err = s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, WorkflowId: id, + StreamName: chasmworkflow.DefaultStreamName, StartOffset: 0, + }, + }) + require.NoError(t, err) + + // Consume the range. This is the task replay will have to reproduce. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + require.Len(t, currentSlice(t, delivered[1]).GetMessages(), 2) + + consumedAt := completedEventWithCursors(t, env.GetHistory(s.ns, + &commonpb.WorkflowExecution{WorkflowId: id, RunId: we.GetRunId()})) + + // Drop the cached context so the next task is served with full history, + // which is the replay path. + env.CloseShard(env.NamespaceID().String(), id) + + signalWorkflow(t, s, id, we.GetRunId()) + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + replayed := sliceForEvent(delivered[2], consumedAt) + require.NotNil(t, replayed, "the replayed task must carry the range recorded at event %d", consumedAt) + require.Equal(t, int64(0), replayed.GetFromOffset()) + require.Equal(t, int64(2), replayed.GetToOffset()) + require.Len(t, replayed.GetMessages(), 2, "the payloads have to come back from the stream") + require.Equal(t, "replay-me-1", string(replayed.GetMessages()[0].GetBody().GetData())) + require.Equal(t, "replay-me-2", string(replayed.GetMessages()[1].GetBody().GetData())) +} + +// completedEventWithCursors returns the id of the first WorkflowTaskCompleted +// event that recorded a non-empty consumed range. +func completedEventWithCursors(t *testing.T, events []*historypb.HistoryEvent) int64 { + t.Helper() + for _, e := range events { + for _, c := range e.GetWorkflowTaskCompletedEventAttributes().GetStreamCursors() { + if c.GetToOffset() > c.GetFromOffset() { + return e.GetEventId() + } + } + } + t.Fatal("no completed event recorded a consumed range") + return 0 +} + +// currentSlice picks the slice for the task about to run. Slices carrying an +// event id belong to tasks being replayed, and a response can hold both. +func currentSlice(t *testing.T, slices []*streampb.StreamSlice) *streampb.StreamSlice { + t.Helper() + for _, s := range slices { + if s.GetWorkflowTaskCompletedEventId() == 0 { + return s + } + } + t.Fatal("no slice for the current task") + return nil +} + +func sliceForEvent(slices []*streampb.StreamSlice, eventID int64) *streampb.StreamSlice { + for _, s := range slices { + if s.GetWorkflowTaskCompletedEventId() == eventID { + return s + } + } + return nil +} + +// The shape the feature exists for: a producer writing off-shard and a workflow +// consuming it, with no relationship between them beyond the stream. +// +// A consuming workflow cannot read another execution's frontier while closing +// its own transaction, so the stream pushes it. The push dirties the consumer, +// whose transaction close then sees it is behind and schedules a workflow task. +// +// The log is read from the stream's own shard rather than the consumer's. +// History nodes are stored per shard, so reading an external stream from the +// consumer's shard finds nothing, and the workflow task then fails to start +// and is dropped rather than reporting anything useful. +func TestWorkflowConsumesAStreamItDoesNotOwn(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + streamID := "external-stream-" + uuid.NewString() + s.create(s.ctx(), t, streamID) + + id := "stream-wf-external-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + we, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + var delivered [][]*streampb.StreamSlice + + //nolint:staticcheck // SA1019: consistent with the other stream tests. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(resp *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + delivered = append(delivered, resp.GetStreamSlices()) + return nil, nil + }, + Logger: env.Logger, + T: t, + } + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + _, err = s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, WorkflowId: id, + StreamId: streamID, StartOffset: 0, + }, + }) + require.NoError(t, err) + + // The subscription has to reach the stream, or truncation could take a + // range the cursor still points at. + desc, err := s.client.DescribeStream(s.ctx(), &streamlib.DescribeStreamRequest{ + FrontendRequest: &streamlib.DescribeStreamInput{Namespace: s.ns, StreamId: streamID}, + }) + require.NoError(t, err) + consumers := desc.GetFrontendResponse().GetState().GetConsumers() + require.Len(t, consumers, 1) + for _, c := range consumers { + require.True(t, c.GetExternal()) + require.True(t, c.GetActive()) + require.Equal(t, id, c.GetWorkflowId()) + } + + // An off-shard producer, unrelated to the workflow. + _, err = s.client.AddMessages(s.ctx(), &streamlib.AddMessagesRequest{ + FrontendRequest: &streamlib.AddMessagesInput{ + Namespace: s.ns, StreamId: streamID, + Messages: []*streamlib.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("from-outside-1")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + {Body: &commonpb.Payload{Data: []byte("from-outside-2")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + }, + }, + }) + require.NoError(t, err) + + // The append alone has to produce a workflow task, with no signal and no + // other traffic against the workflow. + await.RequireTruef(t, func() bool { + events := env.GetHistory(s.ns, &commonpb.WorkflowExecution{WorkflowId: id, RunId: we.GetRunId()}) + scheduled := 0 + for _, e := range events { + if e.GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED { + scheduled++ + } + } + return scheduled >= 2 + }, 20*time.Second, 200*time.Millisecond, "the append must schedule a workflow task on its own") + + // And it must reach a worker. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + got := currentSlice(t, delivered[1]) + require.Equal(t, streamID, got.GetStreamId()) + require.Equal(t, int64(0), got.GetFromOffset()) + require.Equal(t, int64(2), got.GetToOffset()) + require.Len(t, got.GetMessages(), 2) + require.Equal(t, "from-outside-1", string(got.GetMessages()[0].GetBody().GetData())) +} + +// A cursor is workflow state, so a continue-as-new has to carry it. Without +// that the subscription ends silently: the stream keeps its consumer pin and +// keeps pushing to the workflow id, and the successor run has nowhere to put +// what arrives. +func TestSubscriptionSurvivesContinueAsNew(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + streamID := "carried-stream-" + uuid.NewString() + s.create(s.ctx(), t, streamID) + + id := "stream-wf-can-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + var delivered [][]*streampb.StreamSlice + task := 0 + + //nolint:staticcheck // SA1019: consistent with the other stream tests. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(resp *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + delivered = append(delivered, resp.GetStreamSlices()) + task++ + if task == 2 { + // Continue as new, carrying nothing of its own. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ + ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + }, + }, + }}, nil + } + return nil, nil + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + _, err = s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, WorkflowId: id, StreamId: streamID, StartOffset: 0, + }, + }) + require.NoError(t, err) + + // First append, consumed by the original run. + _, err = s.client.AddMessages(s.ctx(), &streamlib.AddMessagesRequest{ + FrontendRequest: &streamlib.AddMessagesInput{ + Namespace: s.ns, StreamId: streamID, + Messages: []*streamlib.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("before-can")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + }, + }, + }) + require.NoError(t, err) + + // Task 2 receives it and continues as new. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + consumed := currentSlice(t, delivered[1]) + require.Equal(t, int64(1), consumed.GetToOffset()) + require.Len(t, consumed.GetMessages(), 1) + + // Drain the successor's first task, which carries nothing new. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + // Append again. Only a carried subscription can deliver this, and it must + // resume at offset 1 rather than replaying from the start. + _, err = s.client.AddMessages(s.ctx(), &streamlib.AddMessagesRequest{ + FrontendRequest: &streamlib.AddMessagesInput{ + Namespace: s.ns, StreamId: streamID, + Messages: []*streamlib.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("after-can")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + }, + }, + }) + require.NoError(t, err) + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + afterCAN := currentSlice(t, delivered[len(delivered)-1]) + require.Equal(t, int64(1), afterCAN.GetFromOffset(), "the successor resumes where its predecessor stopped") + require.Equal(t, int64(2), afterCAN.GetToOffset()) + require.Len(t, afterCAN.GetMessages(), 1) + require.Equal(t, "after-can", string(afterCAN.GetMessages()[0].GetBody().GetData())) +} + +// A workflow subscribing itself, rather than being subscribed out of band. +// +// The command carries only the stream id and a start offset. Everything else +// the cursor needs is looked up by the server: a workflow cannot read another +// execution's collection id or bucket size without doing I/O, and a value it +// carried would be a reading rather than a fact, so it could differ on replay. +func TestWorkflowSubscribesToAStreamItself(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + streamID := "self-sub-stream-" + uuid.NewString() + s.create(s.ctx(), t, streamID) + + id := "stream-wf-selfsub-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + var delivered [][]*streampb.StreamSlice + task := 0 + + //nolint:staticcheck // SA1019: only the deprecated poller can emit this command type. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(resp *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + delivered = append(delivered, resp.GetStreamSlices()) + task++ + if task > 1 { + return nil, nil + } + // The workflow subscribes itself on its first task. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SUBSCRIBE_STREAM, + Attributes: &commandpb.Command_SubscribeStreamCommandAttributes{ + SubscribeStreamCommandAttributes: &commandpb.SubscribeStreamCommandAttributes{ + StreamId: streamID, + StartOffset: 0, + }, + }, + }}, nil + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + require.Empty(t, delivered[0], "nothing is subscribed until this task completes") + + // The subscription has to have reached the stream, or truncation could take + // a range the cursor still points at. + desc, err := s.client.DescribeStream(s.ctx(), &streamlib.DescribeStreamRequest{ + FrontendRequest: &streamlib.DescribeStreamInput{Namespace: s.ns, StreamId: streamID}, + }) + require.NoError(t, err) + require.Len(t, desc.GetFrontendResponse().GetState().GetConsumers(), 1, + "the command must have registered a consumer on the stream") + + // The subscription is recorded once, with the offset the server resolved. + // Without it nothing in History explains why this workflow starts receiving + // stream data, and a command that writes no event desynchronises the + // command-to-event matching every SDK's replay depends on. + subscribed := subscribedEvents(env.GetHistory(s.ns, + &commonpb.WorkflowExecution{WorkflowId: id})) + require.Len(t, subscribed, 1) + require.Equal(t, streamID, subscribed[0].GetStreamId()) + require.Equal(t, int64(0), subscribed[0].GetStartOffset()) + + // An off-shard producer, with no signal to the workflow. + _, err = s.client.AddMessages(s.ctx(), &streamlib.AddMessagesRequest{ + FrontendRequest: &streamlib.AddMessagesInput{ + Namespace: s.ns, StreamId: streamID, + Messages: []*streamlib.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("self-1")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + {Body: &commonpb.Payload{Data: []byte("self-2")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + }, + }, + }) + require.NoError(t, err) + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + got := currentSlice(t, delivered[1]) + require.Equal(t, streamID, got.GetStreamId()) + require.Equal(t, int64(0), got.GetFromOffset()) + require.Equal(t, int64(2), got.GetToOffset()) + require.Len(t, got.GetMessages(), 2) + require.Equal(t, "self-1", string(got.GetMessages()[0].GetBody().GetData())) +} + +func subscribedEvents(events []*historypb.HistoryEvent) []*historypb.WorkflowStreamSubscribedEventAttributes { + var out []*historypb.WorkflowStreamSubscribedEventAttributes + for _, e := range events { + if attrs := e.GetWorkflowStreamSubscribedEventAttributes(); attrs != nil { + out = append(out, attrs) + } + } + return out +} + +// A second subscribe to a stream the workflow already holds registers nothing, +// but it still has to write an event. Every SDK matches issued commands against +// command-generated events by position, so a command producing no event leaves +// the machine that issued it unmatched and puts every later command out of step. +func TestResubscribingStillWritesItsEvent(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + streamID := "resub-stream-" + uuid.NewString() + s.create(s.ctx(), t, streamID) + + id := "stream-wf-resub-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + we, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + subscribe := []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SUBSCRIBE_STREAM, + Attributes: &commandpb.Command_SubscribeStreamCommandAttributes{ + SubscribeStreamCommandAttributes: &commandpb.SubscribeStreamCommandAttributes{ + StreamId: streamID, StartOffset: 0, + }, + }, + }} + + task := 0 + //nolint:staticcheck // SA1019: only the deprecated poller can emit this command type. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(*workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + task++ + if task > 2 { + return nil, nil + } + return subscribe, nil + }, + Logger: env.Logger, + T: t, + } + + // Two tasks, each issuing the same subscribe. The second registers nothing. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + _, err = env.FrontendClient().SignalWorkflowExecution(s.ctx(), &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: s.ns, + WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: id}, + SignalName: "wake", + Identity: "tester", + RequestId: uuid.NewString(), + }) + require.NoError(t, err) + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + events := env.GetHistory(s.ns, &commonpb.WorkflowExecution{WorkflowId: id, RunId: we.GetRunId()}) + var subscribed []*historypb.WorkflowStreamSubscribedEventAttributes + for _, e := range events { + if a := e.GetWorkflowStreamSubscribedEventAttributes(); a != nil { + subscribed = append(subscribed, a) + } + } + require.Len(t, subscribed, 2, "each subscribe command owes an event, even a repeat") + require.Equal(t, streamID, subscribed[1].GetStreamId()) + // The repeat reports where the cursor actually is, not the offset it asked + // for, so a replaying worker reads a fact rather than a request. + require.Equal(t, int64(0), subscribed[1].GetStartOffset()) +} + +// A subscribe followed by another command in the same Workflow Task. Every SDK +// matches issued commands against the events they produced by position, so the +// subscription's event has to sit where its command did. +// +// It used to be written in the flush, which runs after every command, so it +// landed behind the events of commands issued later and the first replay of +// any workflow that subscribed before doing anything else would fail. +func TestStreamSubscribeEventKeepsCommandOrder(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + streamID := "order-src-" + uuid.NewString() + s.create(s.ctx(), t, streamID) + + id := "stream-order-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-order"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + //nolint:staticcheck // SA1019: deprecated poller is the only one that can emit the command. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(*workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + // Subscribe first, then publish. The publish event must come second. + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_SUBSCRIBE_STREAM, + Attributes: &commandpb.Command_SubscribeStreamCommandAttributes{ + SubscribeStreamCommandAttributes: &commandpb.SubscribeStreamCommandAttributes{ + StreamId: streamID, StartOffset: 0, + }, + }, + }, + { + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("after subscribe")}}, + }, + }, + }, + }, + }, nil + }, + Logger: env.Logger, + T: t, + } + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + var order []enumspb.EventType + for _, e := range env.GetHistory(s.ns, &commonpb.WorkflowExecution{WorkflowId: id}) { + switch e.GetEventType() { + case enumspb.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED, + enumspb.EVENT_TYPE_WORKFLOW_STREAM_MESSAGES_ADDED: + order = append(order, e.GetEventType()) + default: + // Every other event is noise for this assertion. + } + } + require.Equal(t, []enumspb.EventType{ + enumspb.EVENT_TYPE_WORKFLOW_STREAM_SUBSCRIBED, + enumspb.EVENT_TYPE_WORKFLOW_STREAM_MESSAGES_ADDED, + }, order, "the events must be in the order their commands were issued") +} diff --git a/tests/stream_publish_cost_test.go b/tests/stream_publish_cost_test.go new file mode 100644 index 00000000000..e76f1ea5538 --- /dev/null +++ b/tests/stream_publish_cost_test.go @@ -0,0 +1,293 @@ +package tests + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + streampb "go.temporal.io/api/stream/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" +) + +// What a history event per publish would cost. +// +// Publishing from workflow code is the one stream command with no history +// event, which is why no SDK can reach it: sdk-core matches commands to events +// positionally, so a command that produces none desynchronises replay. Giving +// it an event fixes that, and the objection is that unlike subscribing, which +// happens once, publishing happens per batch. This measures the per-batch +// price so the trade is decided on a number. +// +// The method is a marginal one. Each arm runs a workflow whose single workflow +// task carries N publish commands and then completes, so history holds the +// fixed opening and closing events plus exactly N publishes. Differencing +// against the N=0 arm cancels the fixed part and leaves the cost of one +// publish. Message size is varied against a fixed batch count to show whether +// payload bytes reach History at all. + +type publishCostArm struct { + name string + batches int + messagesPerBatch int + messageSize int + // Sends the same bytes as Signals instead, which is what a workflow + // streaming today has to do. The comparison is the point of the exercise. + viaSignal bool +} + +type publishCostResult struct { + arm publishCostArm + historyEvents int64 + historyBytes int64 +} + +func (r publishCostResult) messages() int { + return r.arm.batches * r.arm.messagesPerBatch +} + +// The published limits a per-batch event has to be judged against. +const ( + historyCountLimitError = 50 * 1024 + historySizeLimitError = 50 * 1024 * 1024 +) + +func TestStreamPublishHistoryCost(t *testing.T) { + if testing.Short() { + t.Skip("measurement, not a correctness check") + } + + arms := []publishCostArm{ + // The control. Everything is differenced against this. + {name: "control", batches: 0}, + + // Batch count at a fixed message shape: is the cost linear, and what + // is the slope. + {name: "stream-b1", batches: 1, messagesPerBatch: 1, messageSize: 20}, + {name: "stream-b10", batches: 10, messagesPerBatch: 1, messageSize: 20}, + {name: "stream-b100", batches: 100, messagesPerBatch: 1, messageSize: 20}, + {name: "stream-b500", batches: 500, messagesPerBatch: 1, messageSize: 20}, + + // Same batch count, more messages inside each: the per-batch event + // should not notice. + {name: "stream-b100-m10", batches: 100, messagesPerBatch: 10, messageSize: 20}, + + // Same batch count, bigger messages: this is the claim that payloads + // never enter History, stated as a measurement. + {name: "stream-b100-s200", batches: 100, messagesPerBatch: 1, messageSize: 200}, + {name: "stream-b100-s2000", batches: 100, messagesPerBatch: 1, messageSize: 2000}, + + // What the same traffic costs through Signals today. + {name: "signal-b100-s20", batches: 100, messagesPerBatch: 1, messageSize: 20, viaSignal: true}, + {name: "signal-b100-s2000", batches: 100, messagesPerBatch: 1, messageSize: 2000, viaSignal: true}, + } + + results := make([]publishCostResult, 0, len(arms)) + byName := map[string]publishCostResult{} + for _, arm := range arms { + r := runPublishCostArm(t, arm) + results = append(results, r) + byName[arm.name] = r + } + reportPublishCost(t, results) + assertPublishCost(t, byName) +} + +// The report is the deliverable, but the properties it shows are the ones the +// design rests on, so they are asserted rather than left to be eyeballed. +func assertPublishCost(t *testing.T, byName map[string]publishCostResult) { + control := byName["control"] + marginalBytes := func(name string) int64 { + return byName[name].historyBytes - control.historyBytes + } + marginalEvents := func(name string) int64 { + return byName[name].historyEvents - control.historyEvents + } + + // One event per batch, not per message. b100-m10 publishes ten times the + // messages of b100 through the same hundred commands. + require.Equal(t, int64(100), marginalEvents("stream-b100")) + require.Equal(t, int64(100), marginalEvents("stream-b100-m10")) + require.Equal(t, int64(500), marginalEvents("stream-b500")) + + // Ten times the payload must not move History, which is the claim that + // bodies never enter it. Judged against what a leak would cost rather than + // against zero: event sizes wobble by a byte or two on varint widths, while + // 1800 extra bytes per message across a hundred batches would be six + // figures. + sizeDrift := marginalBytes("stream-b100-s2000") - marginalBytes("stream-b100-s200") + require.Less(t, max(sizeDrift, -sizeDrift), int64(2000), + "10x the payload moved the history cost, so payload is reaching History") + + // Ten times the messages through the same batches, within a byte or two of + // noise from varint widths. + require.InDelta(t, marginalBytes("stream-b100"), marginalBytes("stream-b100-m10"), 200, + "cost tracked message count rather than batch count") + + // The comparison that justifies the feature. Same traffic, and Signals are + // the only way to do this today. + require.Less(t, marginalBytes("stream-b100-s2000"), marginalBytes("signal-b100-s2000")/20, + "a stream publish should be far cheaper than the Signal it replaces") +} + +func runPublishCostArm(t *testing.T, arm publishCostArm) publishCostResult { + t.Helper() + env := testcore.NewEnv(t, testcore.WithDisableTestloggerFailure()) + + // Fixed-width regardless of the arm, because the workflow id and the task + // queue name derived from it are both carried in the opening events. Naming + // the arms in there would put ten bytes of arm-name difference into the very + // figure being differenced. + id := "publish-cost-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + ctx := testcore.NewContext() + + _, err := env.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: env.Namespace().String(), + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "publish-cost"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(60 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + body := []byte(strings.Repeat("x", arm.messageSize)) + + // Sent before the first workflow task is polled, so they land in history + // ahead of it and the single task still closes the workflow. + if arm.viaSignal { + for range arm.batches { + payloads := make([]*commonpb.Payload, 0, arm.messagesPerBatch) + for range arm.messagesPerBatch { + payloads = append(payloads, &commonpb.Payload{Data: body}) + } + _, err := env.FrontendClient().SignalWorkflowExecution(ctx, &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: id}, + SignalName: "stream-item", + Input: &commonpb.Payloads{Payloads: payloads}, + Identity: "tester", + RequestId: uuid.NewString(), + }) + require.NoError(t, err) + } + } + + //nolint:staticcheck // SA1019: only the deprecated poller emits raw commands. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: env.Namespace().String(), + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(*workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + commands := make([]*commandpb.Command, 0, arm.batches+1) + if !arm.viaSignal { + for range arm.batches { + messages := make([]*streampb.StreamMessage, 0, arm.messagesPerBatch) + for range arm.messagesPerBatch { + messages = append(messages, &streampb.StreamMessage{ + Body: &commonpb.Payload{Data: body}, + Topic: "progress", + }) + } + commands = append(commands, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: messages, + }, + }, + }) + } + } + return append(commands, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }), nil + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + desc, err := env.FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: id}, + }) + require.NoError(t, err) + + return publishCostResult{ + arm: arm, + historyEvents: desc.GetWorkflowExecutionInfo().GetHistoryLength(), + historyBytes: desc.GetWorkflowExecutionInfo().GetHistorySizeBytes(), + } +} + +func reportPublishCost(t *testing.T, results []publishCostResult) { + var control publishCostResult + for _, r := range results { + if r.arm.name == "control" { + control = r + } + } + + t.Log("Cost of one history event per publish, measured by differencing against an empty run.") + t.Logf("Control: %d events, %d bytes.", control.historyEvents, control.historyBytes) + t.Log("") + t.Log("| arm | batches | msgs | msg size | events | bytes | events/batch | bytes/batch | bytes/msg |") + t.Log("|---|---|---|---|---|---|---|---|---|") + for _, r := range results { + if r.arm.batches == 0 { + continue + } + marginalEvents := r.historyEvents - control.historyEvents + marginalBytes := r.historyBytes - control.historyBytes + perMsg := "n/a" + if r.messages() > 0 { + perMsg = fmt.Sprintf("%.1f", float64(marginalBytes)/float64(r.messages())) + } + t.Logf("| %s | %d | %d | %d | %d | %d | %.2f | %.1f | %s |", + r.arm.name, r.arm.batches, r.messages(), r.arm.messageSize, + r.historyEvents, r.historyBytes, + float64(marginalEvents)/float64(r.arm.batches), + float64(marginalBytes)/float64(r.arm.batches), + perMsg) + } + t.Log("") + + // The ceilings the number has to be read against. An event per batch makes + // the event-count limit the binding one long before the size limit. + for _, r := range results { + if r.arm.batches == 0 || r.arm.viaSignal { + continue + } + marginalBytes := r.historyBytes - control.historyBytes + marginalEvents := r.historyEvents - control.historyEvents + if marginalEvents == 0 { + continue + } + bytesPerEvent := float64(marginalBytes) / float64(marginalEvents) + t.Logf("%s: %.0f bytes/event implies %d batches before the %d-event limit, "+ + "%d before the %dMB size limit", + r.arm.name, bytesPerEvent, + historyCountLimitError, + historyCountLimitError, + int(float64(historySizeLimitError)/bytesPerEvent), + historySizeLimitError/(1024*1024)) + } +} diff --git a/tests/stream_retention_test.go b/tests/stream_retention_test.go new file mode 100644 index 00000000000..d9d91ae604d --- /dev/null +++ b/tests/stream_retention_test.go @@ -0,0 +1,133 @@ +package tests + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + streamlib "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" +) + +// A range a workflow consumed is part of that workflow's recovery: the offsets +// are in its History and a replay is asked to reproduce them from the stream. +// Deleting those messages succeeds immediately and fails much later, during a +// replay nobody is watching, so the stream refuses instead. +func TestTruncationRefusesToDropWhatAConsumerNeedsToReplay(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + streamID := "retained-stream-" + uuid.NewString() + s.create(s.ctx(), t, streamID) + _, err := s.client.AddMessages(s.ctx(), &streamlib.AddMessagesRequest{ + FrontendRequest: &streamlib.AddMessagesInput{ + Namespace: s.ns, StreamId: streamID, + Messages: []*streamlib.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("one")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + {Body: &commonpb.Payload{Data: []byte("two")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + {Body: &commonpb.Payload{Data: []byte("three")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + }, + }, + }) + require.NoError(t, err) + + id := "stream-retention-wf-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + _, err = env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-consumer"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + //nolint:staticcheck // SA1019: consistent with the other stream tests. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(_ *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return nil, nil + }, + Logger: env.Logger, + T: t, + } + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + _, err = s.client.SubscribeWorkflow(s.ctx(), &streamlib.SubscribeWorkflowRequest{ + FrontendRequest: &streamlib.SubscribeWorkflowInput{ + Namespace: s.ns, WorkflowId: id, StreamId: streamID, StartOffset: 0, + }, + }) + require.NoError(t, err) + + // Delivered and recorded, which is what makes these three messages part of + // the workflow's recovery rather than spare capacity. + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + _, err = s.client.TruncateStream(s.ctx(), &streamlib.TruncateStreamRequest{ + FrontendRequest: &streamlib.TruncateStreamInput{ + Namespace: s.ns, StreamId: streamID, NewBaseOffset: 2, + }, + }) + require.ErrorContains(t, err, "still depends on offset 0") + + desc, err := s.client.DescribeStream(s.ctx(), &streamlib.DescribeStreamRequest{ + FrontendRequest: &streamlib.DescribeStreamInput{Namespace: s.ns, StreamId: streamID}, + }) + require.NoError(t, err) + state := desc.GetFrontendResponse().GetState() + require.Equal(t, int64(0), state.GetBaseOffset(), "the messages must still be there") + for _, consumer := range state.GetConsumers() { + require.Equal(t, int64(0), consumer.GetReplayFloor(), + "the floor is where the subscription started, not where it has read to") + } +} + +// The refusal is about a consumer's recovery, so a stream nobody is consuming +// truncates exactly as before. Without this, the fix would be a storage leak +// wearing a correctness argument. +func TestTruncationStillWorksWithNoConsumer(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + streamID := "unwatched-stream-" + uuid.NewString() + s.create(s.ctx(), t, streamID) + _, err := s.client.AddMessages(s.ctx(), &streamlib.AddMessagesRequest{ + FrontendRequest: &streamlib.AddMessagesInput{ + Namespace: s.ns, StreamId: streamID, + Messages: []*streamlib.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("one")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + {Body: &commonpb.Payload{Data: []byte("two")}, Kind: streamlib.STREAM_MESSAGE_KIND_DATA}, + }, + }, + }) + require.NoError(t, err) + + _, err = s.client.TruncateStream(s.ctx(), &streamlib.TruncateStreamRequest{ + FrontendRequest: &streamlib.TruncateStreamInput{ + Namespace: s.ns, StreamId: streamID, NewBaseOffset: 2, + }, + }) + require.NoError(t, err) + + desc, err := s.client.DescribeStream(s.ctx(), &streamlib.DescribeStreamRequest{ + FrontendRequest: &streamlib.DescribeStreamInput{Namespace: s.ns, StreamId: streamID}, + }) + require.NoError(t, err) + require.Equal(t, int64(2), desc.GetFrontendResponse().GetState().GetBaseOffset()) +} diff --git a/tests/stream_test.go b/tests/stream_test.go new file mode 100644 index 00000000000..cb261d4a7ec --- /dev/null +++ b/tests/stream_test.go @@ -0,0 +1,641 @@ +package tests + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + chasmstream "go.temporal.io/server/chasm/lib/stream" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/tests/testcore" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// End-to-end coverage of the native stream path: append through the frontend, +// read back by offset, and the lifecycle transitions around it. This is the +// path the benchmark will eventually compare against the Signal-and-Update +// baseline in streaming_baseline_test.go. + +const streamMaxBatch = chasmstream.MaxMessagesPerBatch + +type streamTestEnv struct { + env *testcore.TestEnv + client streampb.StreamServiceClient + ns string + cleanup []func() +} + +func newStreamTestEnv(t *testing.T) *streamTestEnv { + return newStreamTestEnvFrom(t, testcore.NewEnv(t)) +} + +// newStreamTestEnvFrom lets a test that needs its own env, for example one +// driving the raw task poller, still reach the stream API. +func newStreamTestEnvFrom(t *testing.T, env *testcore.TestEnv) *streamTestEnv { + + conn, err := grpc.NewClient(env.FrontendGRPCAddress(), + grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + env2 := &streamTestEnv{env: env, client: streampb.NewStreamServiceClient(conn), ns: env.Namespace().String()} + t.Cleanup(func() { + for _, c := range env2.cleanup { + c() + } + }) + + return env2 +} + +func (s *streamTestEnv) ctx() context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + s.cleanup = append(s.cleanup, cancel) + return ctx +} + +func (s *streamTestEnv) create(ctx context.Context, t *testing.T, streamID string) { + t.Helper() + _, err := s.client.CreateStream(ctx, &streampb.CreateStreamRequest{ + FrontendRequest: &streampb.CreateStreamInput{Namespace: s.ns, StreamId: streamID}, + }) + require.NoError(t, err) +} + +func (s *streamTestEnv) add( + ctx context.Context, t *testing.T, streamID string, in *streampb.AddMessagesInput, +) (*streampb.AddMessagesOutput, error) { + t.Helper() + in.Namespace = s.ns + in.StreamId = streamID + resp, err := s.client.AddMessages(ctx, &streampb.AddMessagesRequest{FrontendRequest: in}) + if err != nil { + return nil, err + } + return resp.GetFrontendResponse(), nil +} + +func (s *streamTestEnv) poll( + ctx context.Context, t *testing.T, streamID string, from int64, topics ...string, +) *streampb.PollMessagesOutput { + t.Helper() + resp, err := s.client.PollMessages(ctx, &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{ + Namespace: s.ns, StreamId: streamID, FromOffset: from, Topics: topics, + }, + }) + require.NoError(t, err) + return resp.GetFrontendResponse() +} + +func streamMsgs(topic string, bodies ...string) []*streampb.StreamMessage { + out := make([]*streampb.StreamMessage, len(bodies)) + for i, b := range bodies { + out[i] = &streampb.StreamMessage{ + Body: &commonpb.Payload{Data: []byte(b)}, + Topic: topic, + Kind: streampb.STREAM_MESSAGE_KIND_DATA, + } + } + return out +} + +func bodies(msgs []*streampb.StreamMessage) []string { + out := make([]string, len(msgs)) + for i, m := range msgs { + out[i] = string(m.GetBody().GetData()) + } + return out +} + +func streamCtx(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + return ctx +} + +func TestStreamAppendAndRead(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-append-read" + s.create(ctx, t, id) + + first, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "a", "b", "c")}) + require.NoError(t, err) + require.Equal(t, int64(0), first.GetFirstOffset()) + require.Equal(t, int64(3), first.GetNextOffset()) + + second, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "d")}) + require.NoError(t, err) + require.Equal(t, int64(3), second.GetFirstOffset()) + + all := s.poll(ctx, t, id, 0) + require.Equal(t, []string{"a", "b", "c", "d"}, bodies(all.GetMessages())) + require.Equal(t, int64(4), all.GetNextOffset()) + require.Equal(t, int64(4), all.GetHeadOffset()) + require.False(t, all.GetClosed()) + + // A reader owns its cursor, so resuming mid-stream is just another read. + tail := s.poll(ctx, t, id, 2) + require.Equal(t, []string{"c", "d"}, bodies(tail.GetMessages())) + + // Caught up returns empty rather than erroring. + caughtUp := s.poll(ctx, t, id, 4) + require.Empty(t, caughtUp.GetMessages()) + require.Equal(t, int64(4), caughtUp.GetNextOffset()) +} + +func TestStreamManyReadersAreIndependent(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-many-readers" + s.create(ctx, t, id) + + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "a", "b")}) + require.NoError(t, err) + + // No durable per-subscriber state, so reader count is not a state-machine + // concern and there is no equivalent of the 10-subscriber ceiling the + // Signal-and-Update pattern hits. + for range 25 { + got := s.poll(ctx, t, id, 0) + require.Equal(t, []string{"a", "b"}, bodies(got.GetMessages())) + } +} + +func TestStreamProducerDedup(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-dedup" + s.create(ctx, t, id) + + in := &streampb.AddMessagesInput{ + Messages: streamMsgs("", "a", "b"), ProducerId: "p1", Sequence: 1, + } + first, err := s.add(ctx, t, id, in) + require.NoError(t, err) + require.False(t, first.GetDeduplicated()) + + retry, err := s.add(ctx, t, id, &streampb.AddMessagesInput{ + Messages: streamMsgs("", "a", "b"), ProducerId: "p1", Sequence: 1, + }) + require.NoError(t, err) + require.True(t, retry.GetDeduplicated()) + require.Equal(t, first.GetFirstOffset(), retry.GetFirstOffset()) + + // The retry must not have appended a second copy. + got := s.poll(ctx, t, id, 0) + require.Equal(t, []string{"a", "b"}, bodies(got.GetMessages())) + + // Same sequence with different content is a client bug. Returning the + // recorded offsets would report success while dropping the data. + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{ + Messages: streamMsgs("", "different"), ProducerId: "p1", Sequence: 1, + }) + require.ErrorContains(t, err, "different content") +} + +func TestStreamTopicFilter(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-topics" + s.create(ctx, t, id) + + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("tokens", "t1")}) + require.NoError(t, err) + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("tools", "x1")}) + require.NoError(t, err) + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("tokens", "t2")}) + require.NoError(t, err) + + got := s.poll(ctx, t, id, 0, "tokens") + require.Equal(t, []string{"t1", "t2"}, bodies(got.GetMessages())) + // Offsets are global, so a filtered read still advances past what it skipped. + require.Equal(t, int64(3), got.GetNextOffset()) +} + +func TestStreamFinishWritingIsPerProducer(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-finish" + s.create(ctx, t, id) + + _, err := s.client.FinishWriting(ctx, &streampb.FinishWritingRequest{ + FrontendRequest: &streampb.FinishWritingInput{ + Namespace: s.ns, StreamId: id, ProducerId: "p1", + }, + }) + require.NoError(t, err) + + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{ + Messages: streamMsgs("", "a"), ProducerId: "p1", Sequence: 1, + }) + require.Error(t, err) + + // Finishing is per-producer, not a close, so others carry on. + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{ + Messages: streamMsgs("", "b"), ProducerId: "p2", Sequence: 1, + }) + require.NoError(t, err) +} + +func TestStreamCloseAndTruncate(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-lifecycle" + s.create(ctx, t, id) + + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "a", "b", "c")}) + require.NoError(t, err) + + _, err = s.client.TruncateStream(ctx, &streampb.TruncateStreamRequest{ + FrontendRequest: &streampb.TruncateStreamInput{ + Namespace: s.ns, StreamId: id, NewBaseOffset: 1, + }, + }) + require.NoError(t, err) + + // A reader below the floor gets a distinguishable error carrying the floor, + // so it can jump forward rather than fail. + _, err = s.client.PollMessages(ctx, &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{Namespace: s.ns, StreamId: id, FromOffset: 0}, + }) + require.ErrorContains(t, err, "truncated") + + _, err = s.client.CloseStream(ctx, &streampb.CloseStreamRequest{ + FrontendRequest: &streampb.CloseStreamInput{Namespace: s.ns, StreamId: id}, + }) + require.NoError(t, err) + + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "d")}) + require.Error(t, err) + + // Closed is a state a reader observes, not an error, and the data stays + // readable rather than requiring a shutdown handshake with the producer. + got := s.poll(ctx, t, id, 1) + require.True(t, got.GetClosed()) + require.Equal(t, []string{"b", "c"}, bodies(got.GetMessages())) +} + +func TestStreamReadStartingInsideABatch(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-mid-batch" + s.create(ctx, t, id) + + // One batch covering 0..2, a second covering 3. A node is addressed by the + // first offset of its batch, so reading from 2 has to find the node that + // contains it rather than the node whose ID equals it. Getting that wrong + // silently drops the messages before the boundary. + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "a", "b", "c")}) + require.NoError(t, err) + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "d")}) + require.NoError(t, err) + + for from, want := range map[int64][]string{ + 0: {"a", "b", "c", "d"}, + 1: {"b", "c", "d"}, + 2: {"c", "d"}, + 3: {"d"}, + } { + got := s.poll(ctx, t, id, from) + require.Equal(t, want, bodies(got.GetMessages()), "reading from offset %d", from) + require.Equal(t, int64(4), got.GetNextOffset()) + } +} + +func TestStreamBatchSizeIsBounded(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-batch-bound" + s.create(ctx, t, id) + + // The bound is not only admission control: it bounds how far a read has to + // step back to find the node containing an arbitrary offset. + tooMany := make([]string, streamMaxBatch+1) + for i := range tooMany { + tooMany[i] = "x" + } + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", tooMany...)}) + require.ErrorContains(t, err, "exceeds the limit") +} + +func (s *streamTestEnv) pollWait( + ctx context.Context, streamID string, from int64, +) (*streampb.PollMessagesOutput, error) { + resp, err := s.client.PollMessages(ctx, &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{ + Namespace: s.ns, StreamId: streamID, FromOffset: from, WaitNewMessages: true, + }, + }) + if err != nil { + return nil, err + } + return resp.GetFrontendResponse(), nil +} + +func TestStreamLongPollWakesOnAppend(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-longpoll-append" + s.create(ctx, t, id) + + type result struct { + out *streampb.PollMessagesOutput + err error + } + done := make(chan result, 1) + go func() { + out, err := s.pollWait(ctx, id, 0) + done <- result{out, err} + }() + + // The poll is parked on an empty stream; the append is what releases it. + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "a")}) + require.NoError(t, err) + + select { + case r := <-done: + require.NoError(t, r.err) + require.Equal(t, []string{"a"}, bodies(r.out.GetMessages())) + require.Equal(t, int64(1), r.out.GetNextOffset()) + case <-time.After(25 * time.Second): + t.Fatal("long poll did not wake on append") + } +} + +func TestStreamLongPollWakesOnClose(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-longpoll-close" + s.create(ctx, t, id) + + done := make(chan *streampb.PollMessagesOutput, 1) + go func() { + out, err := s.pollWait(ctx, id, 0) + if err == nil { + done <- out + } + }() + + _, err := s.client.CloseStream(ctx, &streampb.CloseStreamRequest{ + FrontendRequest: &streampb.CloseStreamInput{Namespace: s.ns, StreamId: id}, + }) + require.NoError(t, err) + + // Closing has to release a parked reader, otherwise a consumer of a + // finished stream waits out the full timeout for no reason. + select { + case out := <-done: + require.True(t, out.GetClosed()) + require.Empty(t, out.GetMessages()) + case <-time.After(25 * time.Second): + t.Fatal("long poll did not wake on close") + } +} + +func TestStreamLongPollReturnsEmptyOnTimeout(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-longpoll-timeout" + s.create(ctx, t, id) + + // A timeout is an empty response, not an error: the caller polls again + // rather than distinguishing a quiet stream from a failure. + start := time.Now() + out, err := s.pollWait(ctx, id, 0) + require.NoError(t, err) + require.Empty(t, out.GetMessages()) + require.Equal(t, int64(0), out.GetNextOffset()) + require.False(t, out.GetClosed()) + require.Greater(t, time.Since(start), 5*time.Second, "the poll should have parked, not returned immediately") +} + +func TestStreamLongPollReturnsImmediatelyWhenBehind(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-longpoll-behind" + s.create(ctx, t, id) + + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "a", "b")}) + require.NoError(t, err) + + // Waiting is only for a reader that is caught up. One that is behind must + // not be parked behind data that already exists. + start := time.Now() + out, err := s.pollWait(ctx, id, 0) + require.NoError(t, err) + require.Equal(t, []string{"a", "b"}, bodies(out.GetMessages())) + require.Less(t, time.Since(start), 5*time.Second) +} + +func TestStreamCapTruncatesAndReclaims(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-cap" + + _, err := s.client.CreateStream(ctx, &streampb.CreateStreamRequest{ + FrontendRequest: &streampb.CreateStreamInput{ + Namespace: s.ns, StreamId: id, + Lifecycle: &streampb.StreamLifecycle{MaxItems: 4}, + }, + }) + require.NoError(t, err) + + for _, batch := range [][]string{{"a", "b"}, {"c", "d"}, {"e", "f"}} { + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", batch...)}) + require.NoError(t, err) + } + + // Six appended against a cap of four, so the floor advanced without anyone + // asking. Reading from the floor still works and returns exactly what the + // cap retained. + got := s.poll(ctx, t, id, 2) + require.Equal(t, []string{"c", "d", "e", "f"}, bodies(got.GetMessages())) + + // Below the floor is a distinguishable error, not silence. + _, err = s.client.PollMessages(ctx, &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{Namespace: s.ns, StreamId: id, FromOffset: 0}, + }) + require.ErrorContains(t, err, "truncated") +} + +func TestStreamClosedStaysReadable(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-closed-readable" + s.create(ctx, t, id) + + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "a", "b")}) + require.NoError(t, err) + _, err = s.client.CloseStream(ctx, &streampb.CloseStreamRequest{ + FrontendRequest: &streampb.CloseStreamInput{Namespace: s.ns, StreamId: id}, + }) + require.NoError(t, err) + + // Close seals, it does not delete. A consumer can finish draining without + // coordinating a shutdown with the producer, which is the handshake the + // signal-based implementation forces today. + got := s.poll(ctx, t, id, 0) + require.Equal(t, []string{"a", "b"}, bodies(got.GetMessages())) + require.True(t, got.GetClosed()) + + desc, err := s.client.DescribeStream(ctx, &streampb.DescribeStreamRequest{ + FrontendRequest: &streampb.DescribeStreamInput{Namespace: s.ns, StreamId: id}, + }) + require.NoError(t, err) + require.NotNil(t, desc.GetFrontendResponse().GetState().GetCloseTime()) +} + +func TestStreamListStreams(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + + created := []string{"list-a", "list-b", "list-c"} + for _, id := range created { + s.create(ctx, t, id) + } + + // Visibility is written by a task after the create commits, so this is + // eventually consistent by design rather than by accident. + await.RequireTrue(t, func() bool { + resp, err := s.client.ListStreams(ctx, &streampb.ListStreamsRequest{ + FrontendRequest: &streampb.ListStreamsInput{Namespace: s.ns}, + }) + if err != nil { + return false + } + found := make(map[string]bool) + for _, e := range resp.GetFrontendResponse().GetStreams() { + found[e.GetStreamId()] = true + } + for _, id := range created { + if !found[id] { + return false + } + } + return true + }, 20*time.Second, 250*time.Millisecond) +} + +// A capped poll must not read the whole stream to answer. The read is clipped +// to the offsets the caller can be given, so a reader asking for one message +// off a long stream does not pull the rest into the history host on its way. +func TestStreamPollReadsOnlyWhatItReturns(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-poll-cap" + s.create(ctx, t, id) + + for i := range 40 { + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{ + Messages: streamMsgs("", fmt.Sprintf("m%d", i)), + }) + require.NoError(t, err) + } + + got := s.pollMax(ctx, t, id, 0, 1) + require.Equal(t, []string{"m0"}, bodies(got.GetMessages())) + require.Equal(t, int64(1), got.GetNextOffset(), "a capped read advances only over what it gave") + require.Equal(t, int64(40), got.GetHeadOffset(), "the frontier is still reported in full") + + // Paging from there covers the rest, so the cap trims the read and not the stream. + got = s.pollMax(ctx, t, id, got.GetNextOffset(), 10) + require.Equal(t, []string{"m1", "m2", "m3", "m4", "m5", "m6", "m7", "m8", "m9", "m10"}, + bodies(got.GetMessages())) + require.Equal(t, int64(11), got.GetNextOffset()) + + // A filter finding nothing in the page still has to leave the reader able to + // go on, and it must stop at the page rather than scanning to the head. + got = s.pollMaxTopics(ctx, t, id, 0, 5, "nothing-matches-this") + require.Empty(t, got.GetMessages()) + require.Equal(t, int64(5), got.GetNextOffset(), + "a filtered page advanced past its own bound, so the read ran to the head") +} + +// A stream id can be reused. The cached bytes belong to the log, not to the +// name, so a reader of the new stream must never be served the old one's. +func TestStreamPollAfterIdIsReusedServesTheNewStream(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-reused-id" + + s.create(ctx, t, id) + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "old")}) + require.NoError(t, err) + // Read it back so the bytes are certain to be cached before the id is reused. + require.Equal(t, []string{"old"}, bodies(s.poll(ctx, t, id, 0).GetMessages())) + + _, err = s.client.DeleteStream(ctx, &streampb.DeleteStreamRequest{ + FrontendRequest: &streampb.DeleteStreamInput{Namespace: s.ns, StreamId: id}, + }) + require.NoError(t, err) + + s.create(ctx, t, id) + _, err = s.add(ctx, t, id, &streampb.AddMessagesInput{Messages: streamMsgs("", "new")}) + require.NoError(t, err) + + got := s.poll(ctx, t, id, 0) + require.Equal(t, []string{"new"}, bodies(got.GetMessages()), + "a reused id served bytes from the deleted stream") +} + +// A filtered read hands back messages whose offsets are not contiguous, which +// is why the offset rides on the message. A reader cannot count them. +func TestStreamFilteredReadReportsRealOffsets(t *testing.T) { + s := newStreamTestEnv(t) + ctx := streamCtx(t) + const id = "stream-filtered-offsets" + s.create(ctx, t, id) + + for i, topic := range []string{"a", "b", "a", "b", "a"} { + _, err := s.add(ctx, t, id, &streampb.AddMessagesInput{ + Messages: streamMsgs(topic, fmt.Sprintf("m%d", i)), + }) + require.NoError(t, err) + } + + got := s.pollMaxTopics(ctx, t, id, 0, 0, "a") + require.Equal(t, []string{"m0", "m2", "m4"}, bodies(got.GetMessages())) + require.Equal(t, []int64{0, 2, 4}, offsets(got.GetMessages())) +} + +func (s *streamTestEnv) pollMaxTopics( + ctx context.Context, t *testing.T, streamID string, from int64, maxMessages int32, topics ...string, +) *streampb.PollMessagesOutput { + t.Helper() + resp, err := s.client.PollMessages(ctx, &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{ + Namespace: s.ns, StreamId: streamID, FromOffset: from, + MaxMessages: maxMessages, Topics: topics, + }, + }) + require.NoError(t, err) + return resp.GetFrontendResponse() +} + +func (s *streamTestEnv) pollMax( + ctx context.Context, t *testing.T, streamID string, from int64, maxMessages int32, +) *streampb.PollMessagesOutput { + t.Helper() + resp, err := s.client.PollMessages(ctx, &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{ + Namespace: s.ns, StreamId: streamID, FromOffset: from, MaxMessages: maxMessages, + }, + }) + require.NoError(t, err) + return resp.GetFrontendResponse() +} + +func offsets(msgs []*streampb.StreamMessage) []int64 { + out := make([]int64, len(msgs)) + for i, m := range msgs { + out[i] = m.GetOffset() + } + return out +} diff --git a/tests/stream_workflow_test.go b/tests/stream_workflow_test.go new file mode 100644 index 00000000000..6385e0d8907 --- /dev/null +++ b/tests/stream_workflow_test.go @@ -0,0 +1,409 @@ +package tests + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + streampb "go.temporal.io/api/stream/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + streamlib "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" +) + +// Path A: a workflow publishing to a stream it owns. The stream is co-located +// with the workflow, so the frontier advances in the workflow task's own +// commit, and History gets one fixed-size event naming the offset range rather +// than anything that was published. +// +// Driven through the raw task poller rather than an SDK, because emitting a new +// command type does not need one. +func TestStreamWorkflowPublishesWithARangeEvent(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-publish-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + we, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-publisher"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + published := false + // The newer taskpoller cannot emit an arbitrary command type, which is + // the whole point here. + //nolint:staticcheck // SA1019: deprecated poller is the only one that can. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(*workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + if published { + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }}, nil + } + published = true + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("planning")}, Topic: "progress"}, + {Body: &commonpb.Payload{Data: []byte("calling tool")}, Topic: "progress"}, + }, + }, + }, + }}, nil + }, + Logger: env.Logger, + T: t, + } + + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + // One event for the batch, holding the range and none of the payload. Two + // messages were published, so it has to name both of them and stop there. + events := env.GetHistory(s.ns, &commonpb.WorkflowExecution{WorkflowId: id, RunId: we.GetRunId()}) + var added []*historypb.WorkflowStreamMessagesAddedEventAttributes + for _, e := range events { + if a := e.GetWorkflowStreamMessagesAddedEventAttributes(); a != nil { + added = append(added, a) + } + } + require.Len(t, added, 1, "one publish command writes one event") + require.Equal(t, int64(0), added[0].GetFirstOffset()) + require.Equal(t, int64(2), added[0].GetMessageCount()) + require.Equal(t, chasmworkflow.DefaultStreamName, added[0].GetStreamId(), + "an unnamed stream resolves to the default before it is recorded") + + // The bodies stay out of History. Asserted on the serialized event rather + // than on its fields, because a field this test forgot to check would still + // be carrying them. + raw, err := added[0].Marshal() + require.NoError(t, err) + require.NotContains(t, string(raw), "calling tool", + "the event must name the range, never carry the payload") + + // The bodies are readable from outside the workflow, which is the point of + // publishing them: an attached stream has no id, so it is addressed by its + // owner and its name. + poll, err := s.client.PollWorkflowMessages(s.ctx(), &streamlib.PollWorkflowMessagesRequest{ + FrontendRequest: &streamlib.PollWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id, FromOffset: 0, + }, + }) + require.NoError(t, err) + require.Equal(t, []string{"planning", "calling tool"}, bodies(poll.GetFrontendResponse().GetMessages())) + require.Equal(t, []int64{0, 1}, offsets(poll.GetFrontendResponse().GetMessages()), + "each message carries where it sits in the log, so a reader can resume between them") + require.Equal(t, int64(2), poll.GetFrontendResponse().GetNextOffset()) + require.Equal(t, int64(2), poll.GetFrontendResponse().GetHeadOffset()) + + // The topic filter applies to an attached stream the same as to any other, + // and offsets stay assigned over the unfiltered stream. + filtered, err := s.client.PollWorkflowMessages(s.ctx(), &streamlib.PollWorkflowMessagesRequest{ + FrontendRequest: &streamlib.PollWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id, FromOffset: 0, Topics: []string{"nothing-here"}, + }, + }) + require.NoError(t, err) + require.Empty(t, filtered.GetFrontendResponse().GetMessages()) + require.Equal(t, int64(2), filtered.GetFrontendResponse().GetNextOffset(), + "a filtered page still advances the reader") + + // A stream the workflow has not published to reads as an empty one, which + // is what lets a reader attach before the first event. + unwritten, err := s.client.PollWorkflowMessages(s.ctx(), &streamlib.PollWorkflowMessagesRequest{ + FrontendRequest: &streamlib.PollWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id, StreamName: "not-published-to-yet", FromOffset: 0, + }, + }) + require.NoError(t, err) + require.Empty(t, unwritten.GetFrontendResponse().GetMessages()) + require.Equal(t, int64(0), unwritten.GetFrontendResponse().GetHeadOffset()) + require.False(t, unwritten.GetFrontendResponse().GetClosed()) + + // A workflow that does not exist is still an error, so a mistyped owner is + // not mistaken for a stream with nothing in it. + _, err = s.client.PollWorkflowMessages(s.ctx(), &streamlib.PollWorkflowMessagesRequest{ + FrontendRequest: &streamlib.PollWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id + "-does-not-exist", FromOffset: 0, + }, + }) + require.ErrorContains(t, err, "not found") + + // An attached stream still has no standalone id, so the id-addressed read + // must not reach it. + streamID := we.GetRunId() + "/" + chasmworkflow.DefaultStreamName + _, err = s.client.PollMessages(s.ctx(), &streamlib.PollMessagesRequest{ + FrontendRequest: &streamlib.PollMessagesInput{ + Namespace: s.ns, StreamId: streamID, FromOffset: 0, + }, + }) + require.ErrorContains(t, err, "stream not found", + "an attached stream is only reachable through its workflow") +} + +// A reader attached to a workflow's stream before the workflow published +// anything. This is the ordinary case for a UI that opens on a session and +// waits for the agent's first token, so the poll has to park on a stream that +// does not exist yet and wake when the workflow creates it. +func TestStreamWorkflowLongPollWakesOnPublish(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-longpoll-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-publisher"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + type result struct { + out *streamlib.PollMessagesOutput + err error + } + done := make(chan result, 1) + go func() { + resp, err := s.client.PollWorkflowMessages(s.ctx(), &streamlib.PollWorkflowMessagesRequest{ + FrontendRequest: &streamlib.PollWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id, FromOffset: 0, WaitNewMessages: true, + }, + }) + done <- result{resp.GetFrontendResponse(), err} + }() + + //nolint:staticcheck // SA1019: deprecated poller is the only one that can emit the command. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(*workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("first token")}}, + }, + }, + }, + }}, nil + }, + Logger: env.Logger, + T: t, + } + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + select { + case r := <-done: + require.NoError(t, r.err) + require.Equal(t, []string{"first token"}, bodies(r.out.GetMessages())) + require.Equal(t, int64(1), r.out.GetNextOffset()) + case <-time.After(25 * time.Second): + t.Fatal("the parked reader did not wake when the workflow published") + } +} + +// Two producers on one stream: the workflow, whose publishes ride its Workflow +// Task, and something outside it. This is the shape of an agent session, where +// a model activity produces the tokens and the workflow only brackets them. +func TestStreamWorkflowTakesAppendsFromOutsideToo(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-mixed-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-publisher"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + appendOutside := func(body string) *streamlib.AddMessagesOutput { + t.Helper() + resp, err := s.client.AddWorkflowMessages(s.ctx(), &streamlib.AddWorkflowMessagesRequest{ + FrontendRequest: &streamlib.AddWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id, + Messages: []*streamlib.StreamMessage{{ + Body: &commonpb.Payload{Data: []byte(body)}, + Kind: streamlib.STREAM_MESSAGE_KIND_DATA, + }}, + }, + }) + require.NoError(t, err) + return resp.GetFrontendResponse() + } + + // The first writer creates the stream, and here that is not the workflow. + first := appendOutside("token one") + require.Equal(t, int64(0), first.GetFirstOffset()) + + //nolint:staticcheck // SA1019: deprecated poller is the only one that can emit the command. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(*workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("turn ended")}}, + }, + }, + }, + }}, nil + }, + Logger: env.Logger, + T: t, + } + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + // The workflow appended after the outside writer, so it has to continue the + // same log rather than start its own. + require.Equal(t, int64(2), appendOutside("token two").GetFirstOffset()) + + poll, err := s.client.PollWorkflowMessages(s.ctx(), &streamlib.PollWorkflowMessagesRequest{ + FrontendRequest: &streamlib.PollWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id, FromOffset: 0, + }, + }) + require.NoError(t, err) + require.Equal(t, []string{"token one", "turn ended", "token two"}, + bodies(poll.GetFrontendResponse().GetMessages()), + "both producers write one ordered log") + + // The event names the offset the workflow's own publish landed at, which + // only holds if the workflow saw the outside writer's message first. + events := env.GetHistory(s.ns, &commonpb.WorkflowExecution{WorkflowId: id}) + var added []*historypb.WorkflowStreamMessagesAddedEventAttributes + for _, e := range events { + if a := e.GetWorkflowStreamMessagesAddedEventAttributes(); a != nil { + added = append(added, a) + } + } + require.Len(t, added, 1) + require.Equal(t, int64(1), added[0].GetFirstOffset()) +} + +// A reader tailing a workflow that ends has to be released. Nothing can be +// added to a stream inside a closed execution, so a reader parked on it would +// otherwise wait for a message that can never come. +func TestStreamWorkflowStreamClosesWithItsWorkflow(t *testing.T) { + env := testcore.NewEnv(t) + s := newStreamTestEnvFrom(t, env) + + id := "stream-wf-close-" + uuid.NewString() + tq := &taskqueuepb.TaskQueue{Name: id + "-tq", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + _, err := env.FrontendClient().StartWorkflowExecution(s.ctx(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: s.ns, + WorkflowId: id, + WorkflowType: &commonpb.WorkflowType{Name: "stream-publisher"}, + TaskQueue: tq, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: "tester", + }) + require.NoError(t, err) + + describe := func() *streamlib.StreamState { + t.Helper() + resp, err := s.client.DescribeWorkflowStream(s.ctx(), &streamlib.DescribeWorkflowStreamRequest{ + FrontendRequest: &streamlib.DescribeWorkflowStreamInput{ + Namespace: s.ns, WorkflowId: id, + }, + }) + require.NoError(t, err) + return resp.GetFrontendResponse().GetState() + } + + //nolint:staticcheck // SA1019: deprecated poller is the only one that can emit the command. + poller := &testcore.TaskPoller{ + Client: env.FrontendClient(), + Namespace: s.ns, + TaskQueue: tq, + Identity: "tester", + WorkflowTaskHandler: func(*workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_ADD_STREAM_MESSAGES, + Attributes: &commandpb.Command_AddStreamMessagesCommandAttributes{ + AddStreamMessagesCommandAttributes: &commandpb.AddStreamMessagesCommandAttributes{ + Messages: []*streampb.StreamMessage{ + {Body: &commonpb.Payload{Data: []byte("last word")}}, + }, + }, + }, + }, + { + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }, + }, nil + }, + Logger: env.Logger, + T: t, + } + _, err = poller.PollAndProcessWorkflowTask() + require.NoError(t, err) + + require.True(t, describe().GetClosed(), "the stream of a completed workflow reads as closed") + + // Closed does not mean gone. Everything published before the workflow + // ended is still there to be read. + poll, err := s.client.PollWorkflowMessages(s.ctx(), &streamlib.PollWorkflowMessagesRequest{ + FrontendRequest: &streamlib.PollWorkflowMessagesInput{ + Namespace: s.ns, WorkflowId: id, FromOffset: 0, WaitNewMessages: true, + }, + }) + require.NoError(t, err) + require.Equal(t, []string{"last word"}, bodies(poll.GetFrontendResponse().GetMessages())) + require.True(t, poll.GetFrontendResponse().GetClosed()) +} diff --git a/tests/streaming_baseline_test.go b/tests/streaming_baseline_test.go new file mode 100644 index 00000000000..a97817e01b2 --- /dev/null +++ b/tests/streaming_baseline_test.go @@ -0,0 +1,458 @@ +package tests + +import ( + "context" + "fmt" + "os" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/workflowservice/v1" + sdkclient "go.temporal.io/sdk/client" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/tests/testcore" +) + +// Baseline measurements for today's Workflow Streams pattern: an Activity +// batches items into Signals and a consumer long-polls with an Update. AI-198 +// proposes replacing it, and a replacement cannot be justified without knowing +// what the current pattern actually costs. +// +// The two dimensions that matter are the flush interval and the subscriber +// count. The shipped guidance is a ~2s flush; the Native Streams 1-pager asks +// for 100ms to be practical. Running both shows the cost of closing that gap +// under the current design, which is the number the September check-in needs. +// +// The full matrix takes minutes, so it is opt-in via TEMPORAL_STREAM_BENCH=1. +// Without it a short configuration still runs, which keeps the harness honest +// without slowing the suite. + +const ( + streamBatchSignal = "stream_batch" + streamDoneSignal = "stream_done" + streamPollUpdate = "poll_events" + streamMessageSize = 20 +) + +type streamBaselineParams struct { + name string + flushInterval time.Duration + subscribers int + messageRate int // messages per second + duration time.Duration +} + +type streamBaselineResult struct { + params streamBaselineParams + + messagesSent int + messagesReceived int + pollRejections int64 + + historyBytes int64 + historyEvents int64 + // Set when the workload has no workflow at all, so the history columns are + // an absence rather than a measurement. Rendering them as 0.00 next to a + // measured figure reads as a comparison that was never made. + historyNotApplicable bool + + persistenceRequests int64 + persistenceByOp map[string]int64 + + latencyP50 time.Duration + latencyP99 time.Duration + + // Set when the run could not complete, for example because the workflow + // exceeded a history limit. That is itself a result worth reporting. + failure string +} + +// streamBaselineWorkflow mirrors the shipped pattern: Signals carry batches in, +// an Update long-polls them back out, and the workflow holds the buffer. +func streamBaselineWorkflow(ctx workflow.Context) error { + var buffer []string + done := false + + err := workflow.SetUpdateHandler(ctx, streamPollUpdate, + func(ctx workflow.Context, lastSeen int) ([]string, error) { + // The shape that makes this expensive: every poll is a durable + // state transition, even when it returns nothing new. + if err := workflow.Await(ctx, func() bool { + return len(buffer) > lastSeen || done + }); err != nil { + return nil, err + } + if lastSeen >= len(buffer) { + return nil, nil + } + out := make([]string, len(buffer)-lastSeen) + copy(out, buffer[lastSeen:]) + return out, nil + }) + if err != nil { + return err + } + + batches := workflow.GetSignalChannel(ctx, streamBatchSignal) + finish := workflow.GetSignalChannel(ctx, streamDoneSignal) + + for !done { + sel := workflow.NewSelector(ctx) + sel.AddReceive(batches, func(c workflow.ReceiveChannel, _ bool) { + var batch []string + c.Receive(ctx, &batch) + buffer = append(buffer, batch...) + }) + sel.AddReceive(finish, func(c workflow.ReceiveChannel, _ bool) { + c.Receive(ctx, nil) + done = true + }) + sel.Select(ctx) + } + + // Let any parked pollers observe the terminal state before exiting. + return workflow.Await(ctx, func() bool { return true }) +} + +func TestStreamingBaseline(t *testing.T) { + matrix := shortStreamBaselineMatrix() + if os.Getenv("TEMPORAL_STREAM_BENCH") == "1" { + matrix = fullStreamBaselineMatrix() + } + + results := make([]streamBaselineResult, 0, len(matrix)) + for _, p := range matrix { + t.Run(p.name, func(t *testing.T) { + results = append(results, runStreamBaseline(t, p)) + }) + } + reportStreamBaseline(t, results) +} + +func shortStreamBaselineMatrix() []streamBaselineParams { + return []streamBaselineParams{ + {name: "flush2s_sub1", flushInterval: 2 * time.Second, subscribers: 1, messageRate: 40, duration: 6 * time.Second}, + {name: "flush100ms_sub1", flushInterval: 100 * time.Millisecond, subscribers: 1, messageRate: 40, duration: 6 * time.Second}, + } +} + +func fullStreamBaselineMatrix() []streamBaselineParams { + var out []streamBaselineParams + for _, flush := range []time.Duration{2 * time.Second, 100 * time.Millisecond} { + for _, subs := range []int{1, 5, 25} { + out = append(out, streamBaselineParams{ + name: fmt.Sprintf("flush%s_sub%d", flush, subs), + flushInterval: flush, + subscribers: subs, + messageRate: 40, + duration: 50 * time.Second, + }) + } + } + return out +} + +func runStreamBaseline(t *testing.T, p streamBaselineParams) streamBaselineResult { + // Dedicated cluster so competing test load does not distort the timings. + // Metric capture is namespace-scoped because persistence_requests carries a + // namespace tag, which also attributes the counts to this workload rather + // than to cluster background traffic. + // + // Testlogger failure is off because a sustained-load run torn down while + // queues are still draining always logs shard-status errors on shutdown. + // The cost is losing the safety net that would catch a genuine server error + // during the run, so treat an anomalous result as a reason to re-run with + // the option removed rather than trusting it. + env := testcore.NewEnv(t, testcore.WithDisableTestloggerFailure()) + res := streamBaselineResult{params: p, persistenceByOp: map[string]int64{}} + + env.SdkWorker().RegisterWorkflow(streamBaselineWorkflow) + + ctx, cancel := context.WithTimeout(context.Background(), p.duration+2*time.Minute) + defer cancel() + + wfID := env.Tv().WorkflowID() + run, err := env.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: wfID, + TaskQueue: env.WorkerTaskQueue(), + }, streamBaselineWorkflow) + require.NoError(t, err) + + // Started after workflow creation so cluster, namespace, and start costs do + // not inflate the per-message figures. Both designs pay those equally. + capture := env.StartNamespaceMetricCapture() + + sentAt := &sync.Map{} // message sequence -> generation time + var receivedTotal atomic.Int64 + var pollRejections atomic.Int64 + var consumers sync.WaitGroup + consumerLatencies := make([][]time.Duration, p.subscribers) + consumerCounts := make([]int, p.subscribers) + consumerCtx, stopConsumers := context.WithCancel(ctx) + defer stopConsumers() + + for i := range p.subscribers { + consumers.Add(1) + go func(idx int) { + defer consumers.Done() + lat, n := runStreamConsumer(consumerCtx, env, wfID, run.GetRunID(), sentAt, &receivedTotal, &pollRejections) + consumerLatencies[idx] = lat + consumerCounts[idx] = n + }(i) + } + + res.messagesSent = runStreamProducer(ctx, t, env, wfID, run.GetRunID(), p, sentAt, &res) + + // Drain, then stop the consumers, and only then let the workflow finish, so + // that no consumer is ever polling an execution that is completing. One that + // is counts its own doomed polls as rejections and its client backoff as + // delivery latency, which measures the harness rather than the pattern. The + // workflow finishes last so the history numbers below are final rather than a + // mid-flight snapshot. A cell that fails to drain is reported rather than + // failed: hitting a limit is a real property of this pattern and is part of + // what the benchmark is measuring. + want := int64(res.messagesSent) * int64(p.subscribers) + if !waitForDrain(ctx, &receivedTotal, want, 15*time.Second) { + t.Logf("drained %d of %d expected deliveries before timeout", receivedTotal.Load(), want) + } + stopConsumers() + consumers.Wait() + require.NoError(t, env.SdkClient().SignalWorkflow(ctx, wfID, run.GetRunID(), streamDoneSignal, nil)) + if err := run.Get(ctx, nil); err != nil { + t.Logf("workflow did not complete cleanly: %v", err) + } + + var all []time.Duration + for i := range p.subscribers { + all = append(all, consumerLatencies[i]...) + res.messagesReceived += consumerCounts[i] + } + res.pollRejections = pollRejections.Load() + res.latencyP50 = percentile(all, 0.50) + res.latencyP99 = percentile(all, 0.99) + + desc, err := env.FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + Execution: env.Tv().WithWorkflowID(wfID).WorkflowExecution(), + }) + if err == nil { + res.historyBytes = desc.GetWorkflowExecutionInfo().GetHistorySizeBytes() + res.historyEvents = desc.GetWorkflowExecutionInfo().GetHistoryLength() + } + + for _, rec := range capture.Metric(metrics.PersistenceRequests.Name()) { + res.persistenceRequests += recordingCount(rec) + if op, ok := rec.Tags["operation"]; ok { + res.persistenceByOp[op] += recordingCount(rec) + } + } + + return res +} + +func runStreamProducer( + ctx context.Context, + t *testing.T, + env *testcore.TestEnv, + wfID, runID string, + p streamBaselineParams, + sentAt *sync.Map, + res *streamBaselineResult, +) int { + payload := make([]byte, streamMessageSize) + for i := range payload { + payload[i] = 'x' + } + + // Messages are generated continuously and flushed on the interval, which is + // what an EventBatcher does. Latency is stamped at generation, not at + // flush: the time an item waits in the batcher is the dominant cost of a + // long flush interval, and stamping at flush would hide it entirely. + genTicker := time.NewTicker(time.Second / time.Duration(p.messageRate)) + defer genTicker.Stop() + flushTicker := time.NewTicker(p.flushInterval) + defer flushTicker.Stop() + deadline := time.Now().Add(p.duration) + + seq := 0 + var pending []string + + flush := func() bool { + if len(pending) == 0 { + return true + } + batch := pending + pending = nil + if err := env.SdkClient().SignalWorkflow(ctx, wfID, runID, streamBatchSignal, batch); err != nil { + // A history or signal limit is a legitimate outcome for this + // pattern, not a broken harness. Record it and stop. + res.failure = err.Error() + t.Logf("producer stopped after %d messages: %v", seq, err) + return false + } + return true + } + + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return seq + case <-genTicker.C: + sentAt.Store(seq, time.Now()) + pending = append(pending, fmt.Sprintf("%d:%s", seq, payload)) + seq++ + case <-flushTicker.C: + if !flush() { + return seq + } + } + } + flush() + return seq +} + +func runStreamConsumer( + ctx context.Context, + env *testcore.TestEnv, + wfID, runID string, + sentAt *sync.Map, + receivedTotal *atomic.Int64, + rejections *atomic.Int64, +) ([]time.Duration, int) { + var latencies []time.Duration + var batch []string + lastSeen := 0 + + for ctx.Err() == nil { + handle, err := env.SdkClient().UpdateWorkflow(ctx, sdkclient.UpdateWorkflowOptions{ + WorkflowID: wfID, + RunID: runID, + UpdateName: streamPollUpdate, + Args: []any{lastSeen}, + WaitForStage: sdkclient.WorkflowUpdateStageCompleted, + }) + if err == nil { + err = handle.Get(ctx, &batch) + } + if err != nil { + // A rejected poll is a measurement, not a reason to stop. Killing + // the consumer here would report a server limit as consumer + // slowness, which is a different and much less useful claim. + rejections.Add(1) + select { + case <-ctx.Done(): + return latencies, lastSeen + case <-time.After(50 * time.Millisecond): + } + continue + } + received := time.Now() + for range batch { + if v, ok := sentAt.Load(lastSeen); ok { + latencies = append(latencies, received.Sub(v.(time.Time))) + } + lastSeen++ + receivedTotal.Add(1) + } + } + return latencies, lastSeen +} + +// historyPerMsg renders a per-message history figure, keeping a workload with +// no workflow distinct from one that measured zero. +func (r streamBaselineResult) historyPerMsg(v int64) string { + if r.historyNotApplicable { + return "no workflow" + } + if r.messagesSent == 0 { + return "n/a" + } + return fmt.Sprintf("%.2f", float64(v)/float64(r.messagesSent)) +} + +// waitForDrain polls until every consumer has caught up or the deadline passes. +// It reports rather than asserts, because a cell that cannot drain is a result. +func waitForDrain(ctx context.Context, got *atomic.Int64, want int64, timeout time.Duration) bool { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + deadline := time.After(timeout) + for { + if got.Load() >= want { + return true + } + select { + case <-ctx.Done(): + return false + case <-deadline: + return false + case <-ticker.C: + } + } +} + +func recordingCount(rec *metricstest.CapturedRecording) int64 { + switch v := rec.Value.(type) { + case int64: + return v + case float64: + return int64(v) + default: + return 0 + } +} + +func percentile(d []time.Duration, q float64) time.Duration { + if len(d) == 0 { + return 0 + } + sorted := make([]time.Duration, len(d)) + copy(sorted, d) + slices.Sort(sorted) + idx := int(float64(len(sorted)-1) * q) + return sorted[idx] +} + +func reportStreamBaseline(t *testing.T, results []streamBaselineResult) { + // Emitted as markdown so the numbers can go straight into the design docs + // without being retyped, which is how transcription errors get in. + t.Log("Workflow Streams baseline: Signals in, polling Update out") + t.Log("") + t.Log("| scenario | msgs | delivered | rejected polls | hist events/msg | hist bytes/msg | persist ops/msg | p50 | p99 |") + t.Log("|---|---|---|---|---|---|---|---|---|") + for _, r := range results { + perMsg := func(v int64) string { + if r.messagesSent == 0 { + return "n/a" + } + return fmt.Sprintf("%.2f", float64(v)/float64(r.messagesSent)) + } + t.Logf("| %s | %d | %d | %d | %s | %s | %s | %s | %s |", + r.params.name, r.messagesSent, r.messagesReceived, r.pollRejections, + r.historyPerMsg(r.historyEvents), r.historyPerMsg(r.historyBytes), + perMsg(r.persistenceRequests), + r.latencyP50.Round(time.Millisecond), r.latencyP99.Round(time.Millisecond)) + } + t.Log("") + for _, r := range results { + if r.failure != "" { + t.Logf("%s did not complete: %s", r.params.name, r.failure) + } + if r.messagesSent > 0 && r.messagesReceived < r.messagesSent*r.params.subscribers { + t.Logf("%s under-delivered: %d of %d expected", + r.params.name, r.messagesReceived, r.messagesSent*r.params.subscribers) + } + } + t.Log("Totals are absolute, not rates: persist ops exclude cluster and workflow start.") + t.Log("Rejected polls are bounded by history.maxInFlightUpdates (default 10) and") + t.Log("history.maxTotalUpdates (default 2000), both per workflow execution.") + for _, r := range results { + t.Logf("%s raw: events=%d bytes=%d persistOps=%d byOp=%v", + r.params.name, r.historyEvents, r.historyBytes, r.persistenceRequests, r.persistenceByOp) + } +} diff --git a/tests/streaming_native_test.go b/tests/streaming_native_test.go new file mode 100644 index 00000000000..a7e400caee8 --- /dev/null +++ b/tests/streaming_native_test.go @@ -0,0 +1,260 @@ +package tests + +import ( + "context" + "fmt" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + streampb "go.temporal.io/server/chasm/lib/stream/gen/streampb/v1" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/tests/testcore" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// The native-stream half of the comparison. It deliberately shares the workload +// shape, latency stamping, and metric capture with the Signal-and-Update +// baseline in streaming_baseline_test.go, because a benchmark whose two halves +// generate load differently measures the harness rather than the design. + +func runNativeStream(t *testing.T, p streamBaselineParams) streamBaselineResult { + env := testcore.NewEnv(t, testcore.WithDisableTestloggerFailure()) + res := streamBaselineResult{params: p, persistenceByOp: map[string]int64{}} + + conn, err := grpc.NewClient(env.FrontendGRPCAddress(), + grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer func() { _ = conn.Close() }() + client := streampb.NewStreamServiceClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), p.duration+2*time.Minute) + defer cancel() + + ns := env.Namespace().String() + streamID := fmt.Sprintf("bench-%s", p.name) + created, err := client.CreateStream(ctx, &streampb.CreateStreamRequest{ + FrontendRequest: &streampb.CreateStreamInput{Namespace: ns, StreamId: streamID}, + }) + require.NoError(t, err) + runID := created.GetFrontendResponse().GetRunId() + + // Started after creation so cluster and namespace setup do not inflate the + // per-message figures, matching the baseline. + capture := env.StartNamespaceMetricCapture() + + sentAt := &sync.Map{} + var receivedTotal atomic.Int64 + var pollRejections atomic.Int64 + + consumerCtx, stopConsumers := context.WithCancel(ctx) + defer stopConsumers() + var consumers sync.WaitGroup + latencies := make([][]time.Duration, p.subscribers) + counts := make([]int, p.subscribers) + + for i := range p.subscribers { + consumers.Add(1) + go func(idx int) { + defer consumers.Done() + latencies[idx], counts[idx] = runNativeConsumer( + consumerCtx, client, ns, streamID, runID, sentAt, &receivedTotal, &pollRejections) + }(i) + } + + res.messagesSent = runNativeProducer(ctx, t, client, ns, streamID, runID, p, sentAt, &res) + + want := int64(res.messagesSent) * int64(p.subscribers) + if !waitForDrain(ctx, &receivedTotal, want, 15*time.Second) { + t.Logf("drained %d of %d expected deliveries before timeout", receivedTotal.Load(), want) + } + stopConsumers() + consumers.Wait() + + var all []time.Duration + for i := range p.subscribers { + all = append(all, latencies[i]...) + res.messagesReceived += counts[i] + } + res.pollRejections = pollRejections.Load() + res.latencyP50 = percentile(all, 0.50) + res.latencyP99 = percentile(all, 0.99) + + // This workload has no workflow: the producer and every consumer are external + // gRPC clients. So there is no execution to describe and no history figure to + // report, which is not the same claim as a measured zero and must not be + // rendered as one. What a publish from workflow code costs in history is + // measured against a control in stream_publish_cost_test.go. + res.historyNotApplicable = true + + for _, rec := range capture.Metric(metrics.PersistenceRequests.Name()) { + res.persistenceRequests += recordingCount(rec) + if op, ok := rec.Tags["operation"]; ok { + res.persistenceByOp[op] += recordingCount(rec) + } + } + return res +} + +func runNativeProducer( + ctx context.Context, + t *testing.T, + client streampb.StreamServiceClient, + ns, streamID, runID string, + p streamBaselineParams, + sentAt *sync.Map, + res *streamBaselineResult, +) int { + payload := make([]byte, streamMessageSize) + for i := range payload { + payload[i] = 'x' + } + + genTicker := time.NewTicker(time.Second / time.Duration(p.messageRate)) + defer genTicker.Stop() + flushTicker := time.NewTicker(p.flushInterval) + defer flushTicker.Stop() + deadline := time.Now().Add(p.duration) + + seq := 0 + var pending []*streampb.StreamMessage + sequence := int64(0) + + flush := func() bool { + if len(pending) == 0 { + return true + } + batch := pending + pending = nil + sequence++ + _, err := client.AddMessages(ctx, &streampb.AddMessagesRequest{ + FrontendRequest: &streampb.AddMessagesInput{ + Namespace: ns, StreamId: streamID, RunId: runID, Messages: batch, + ProducerId: "bench", Sequence: sequence, + }, + }) + if err != nil { + res.failure = err.Error() + t.Logf("producer stopped after %d messages: %v", seq, err) + return false + } + return true + } + + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return seq + case <-genTicker.C: + sentAt.Store(seq, time.Now()) + pending = append(pending, &streampb.StreamMessage{ + Body: &commonpb.Payload{Data: payload}, + Kind: streampb.STREAM_MESSAGE_KIND_DATA, + }) + seq++ + case <-flushTicker.C: + if !flush() { + return seq + } + } + } + flush() + return seq +} + +func runNativeConsumer( + ctx context.Context, + client streampb.StreamServiceClient, + ns, streamID, runID string, + sentAt *sync.Map, + receivedTotal *atomic.Int64, + rejections *atomic.Int64, +) ([]time.Duration, int) { + var out []time.Duration + lastSeen := int64(0) + + for ctx.Err() == nil { + resp, err := client.PollMessages(ctx, &streampb.PollMessagesRequest{ + FrontendRequest: &streampb.PollMessagesInput{ + Namespace: ns, StreamId: streamID, RunId: runID, + FromOffset: lastSeen, WaitNewMessages: true, + }, + }) + if err != nil { + rejections.Add(1) + select { + case <-ctx.Done(): + return out, int(lastSeen) + case <-time.After(50 * time.Millisecond): + } + continue + } + received := time.Now() + fr := resp.GetFrontendResponse() + for range fr.GetMessages() { + if v, ok := sentAt.Load(int(lastSeen)); ok { + out = append(out, received.Sub(v.(time.Time))) + } + lastSeen++ + receivedTotal.Add(1) + } + if next := fr.GetNextOffset(); next > lastSeen { + lastSeen = next + } + } + return out, int(lastSeen) +} + +// TestStreamingComparison runs both designs over the same workload and reports +// them together. This is the artifact the go/no-go decision needs. +func TestStreamingComparison(t *testing.T) { + matrix := shortStreamBaselineMatrix() + if os.Getenv("TEMPORAL_STREAM_BENCH") == "1" { + matrix = fullStreamBaselineMatrix() + } + + var baseline, native []streamBaselineResult + for _, p := range matrix { + t.Run("baseline/"+p.name, func(t *testing.T) { + baseline = append(baseline, runStreamBaseline(t, p)) + }) + t.Run("native/"+p.name, func(t *testing.T) { + native = append(native, runNativeStream(t, p)) + }) + } + + t.Log("Workflow Streams (Signals in, polling Update out) versus native streams") + t.Log("") + t.Log("| scenario | design | msgs | delivered | rejected | wf hist bytes/msg | persist ops/msg | p50 | p99 |") + t.Log("|---|---|---|---|---|---|---|---|---|") + for i := range baseline { + logComparisonRow(t, "signals+update", baseline[i]) + logComparisonRow(t, "native", native[i]) + } + + t.Log("") + for i := range baseline { + t.Logf("%s signals+update raw: persistOps=%d byOp=%v", + baseline[i].params.name, baseline[i].persistenceRequests, baseline[i].persistenceByOp) + t.Logf("%s native raw: persistOps=%d byOp=%v", + native[i].params.name, native[i].persistenceRequests, native[i].persistenceByOp) + } +} + +func logComparisonRow(t *testing.T, design string, r streamBaselineResult) { + perMsg := func(v int64) string { + if r.messagesSent == 0 { + return "n/a" + } + return fmt.Sprintf("%.2f", float64(v)/float64(r.messagesSent)) + } + t.Logf("| %s | %s | %d | %d | %d | %s | %s | %s | %s |", + r.params.name, design, r.messagesSent, r.messagesReceived, r.pollRejections, + r.historyPerMsg(r.historyBytes), perMsg(r.persistenceRequests), + r.latencyP50.Round(time.Millisecond), r.latencyP99.Round(time.Millisecond)) +} diff --git a/tests/testcore/history_task_recorder.go b/tests/testcore/history_task_recorder.go index 380fcdd8ecd..83fb46b1e67 100644 --- a/tests/testcore/history_task_recorder.go +++ b/tests/testcore/history_task_recorder.go @@ -619,3 +619,27 @@ func (r *HistoryTaskRecorder) GetAllHistoryTreeBranches( ) (*persistence.GetAllHistoryTreeBranchesResponse, error) { return r.delegate.GetAllHistoryTreeBranches(ctx, request) } + +// Stream log pass-throughs. This recorder only watches task generation, and a +// stream log write generates none. + +func (r *HistoryTaskRecorder) AppendStreamLog( + ctx context.Context, + request *persistence.InternalAppendStreamLogRequest, +) error { + return r.delegate.AppendStreamLog(ctx, request) +} + +func (r *HistoryTaskRecorder) ReadStreamLog( + ctx context.Context, + request *persistence.InternalReadStreamLogRequest, +) (*persistence.InternalReadStreamLogResponse, error) { + return r.delegate.ReadStreamLog(ctx, request) +} + +func (r *HistoryTaskRecorder) DeleteStreamLogBucket( + ctx context.Context, + request *persistence.InternalDeleteStreamLogBucketRequest, +) error { + return r.delegate.DeleteStreamLogBucket(ctx, request) +}