diff --git a/examples/common/common.go b/examples/common/common.go index 6461aa1..77dc3ad 100644 --- a/examples/common/common.go +++ b/examples/common/common.go @@ -19,7 +19,6 @@ import ( "strings" "github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/dsdk" - "github.com/go-chi/chi/v5" ) const ( @@ -36,29 +35,9 @@ type TokenResponse struct { Endpoint string `json:"url"` } -// NewSignalingServer creates and returns a new HTTP server configured with dataplane signaling endpoints. -func NewSignalingServer(sdkApi *dsdk.DataPlaneApi, port int) *http.Server { - r := chi.NewRouter() - r.Post("/dataflows/start", sdkApi.Start) - r.Post("/dataflows/{id}/started", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.StartById(writer, request, id) - }) - r.Post("/dataflows/prepare", sdkApi.Prepare) - r.Post("/dataflows/{id}/terminate", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.Terminate(id, writer, request) - }) - r.Post("/dataflows/{id}/suspend", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.Suspend(id, writer, request) - }) - r.Get("/dataflows/{id}/status", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.Status(id, writer, request) - }) - - return &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: r} +// NewSignalingServer creates and returns a new HTTP server serving the data plane signaling endpoints under /v1. +func NewSignalingServer(sdk *dsdk.DataPlaneSDK, port int) *http.Server { + return &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: dsdk.NewHandler(sdk)} } // NewDataServer creates and initializes a new HTTP server with a specified port and request handler. diff --git a/examples/controlplane/controlplane.go b/examples/controlplane/controlplane.go index a62fceb..0e1bc2f 100644 --- a/examples/controlplane/controlplane.go +++ b/examples/controlplane/controlplane.go @@ -17,8 +17,8 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" - "net/url" "time" "github.com/eclipse-dataplane-core/dataplane-sdk-go/examples/common" @@ -27,216 +27,129 @@ import ( ) const ( - startUrl = "http://localhost:%d/dataflows/start" - terminateUrl = "http://localhost:%d/dataflows/%s/terminate" - consumerPrepareURL = "http://localhost:%d/dataflows/prepare" - providerCallbackURL = "http://provider.com/dp/callback" - contentType = "Content-Type" - jsonContentType = "application/json" + signalingUrl = "http://localhost:%d" + dsdk.DefaultBasePath + "/dataflows" + contentType = "Content-Type" + jsonContentType = "application/json" + consumerID = "did:web:consumer.com" + providerID = "did:web:provider.com" ) // ControlPlaneSimulator simulates control plane interactions between a consumer and provider and drives their respective data planes. type ControlPlaneSimulator struct { - consumerDataPlane string - providerDataPlane string + client *http.Client } func NewSimulator() (*ControlPlaneSimulator, error) { - return &ControlPlaneSimulator{}, nil + return &ControlPlaneSimulator{client: &http.Client{Timeout: 30 * time.Second}}, nil } +// ProviderStart sends a start message to the provider data plane. For push transfers da is the consumer's DataAddress; +// for pull transfers it must be nil and the provider's DataAddress is returned. func (c *ControlPlaneSimulator) ProviderStart(ctx context.Context, - processID string, + dataFlowID string, agreementID string, datasetID string, + profile string, da *dsdk.DataAddress) (*dsdk.DataAddress, error) { - callbackURL, _ := url.Parse(providerCallbackURL) startMessage := dsdk.DataFlowStartMessage{ - DataFlowBaseMessage: dsdk.DataFlowBaseMessage{ - MessageID: uuid.NewString(), - AgreementID: agreementID, - DatasetID: datasetID, - ProcessID: processID, - DataspaceContext: "dscontext", - CounterPartyID: "did:web:consumer.com", - ParticipantID: "did:web:provider.com", - CallbackAddress: dsdk.CallbackURL(*callbackURL), - TransferType: dsdk.TransferType{DestinationType: "custom", FlowType: dsdk.Pull}, - DataAddress: da, - }, + DataFlowBaseMessage: newBaseMessage(dataFlowID, agreementID, datasetID, profile, providerID, consumerID), + DataAddress: da, } - - serialized, err := json.Marshal(startMessage) - if err != nil { - return nil, fmt.Errorf("failed to marshal start message: %w", err) - } - - // Create the request - providerSignalingUrl := fmt.Sprintf(startUrl, common.ProviderSignalingPort) - req, err := http.NewRequestWithContext(ctx, "POST", providerSignalingUrl, bytes.NewBuffer(serialized)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set(contentType, jsonContentType) - - client := &http.Client{ - Timeout: 30 * time.Second, - } - - resp, err := client.Do(req) + status, err := c.send(ctx, fmt.Sprintf(signalingUrl+"/start", common.ProviderSignalingPort), startMessage) if err != nil { return nil, fmt.Errorf("start request failed: %w", err) } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - var message dsdk.DataFlowResponseMessage - if err := json.NewDecoder(resp.Body).Decode(&message); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - return message.DataAddress, nil + return status.DataAddress, nil } -func (c *ControlPlaneSimulator) ConsumerStart(ctx context.Context, processID string, source *dsdk.DataAddress) error { - callbackURL, _ := url.Parse(providerCallbackURL) - startMessage := dsdk.DataFlowStartMessage{ - DataFlowBaseMessage: dsdk.DataFlowBaseMessage{ - MessageID: uuid.NewString(), - ProcessID: processID, - AgreementID: uuid.NewString(), - DataspaceContext: "dscontext", - ParticipantID: "did:web:consumer.com", - CounterPartyID: "did:web:provider.com", - CallbackAddress: dsdk.CallbackURL(*callbackURL), - TransferType: dsdk.TransferType{DestinationType: "custom", FlowType: dsdk.Pull}, - DataAddress: source, - }, - } - - serialized, err := json.Marshal(startMessage) - if err != nil { - return fmt.Errorf("failed to marshal start message: %w", err) +// ConsumerPrepare sends a prepare message to the consumer data plane. For push transfers the consumer's DataAddress +// is returned. +func (c *ControlPlaneSimulator) ConsumerPrepare(ctx context.Context, dataFlowID string, agreementID string, datasetID string, profile string) (*dsdk.DataAddress, error) { + prepareMessage := dsdk.DataFlowPrepareMessage{ + DataFlowBaseMessage: newBaseMessage(dataFlowID, agreementID, datasetID, profile, consumerID, providerID), } - - // Create the request - consumerSignalingUrl := fmt.Sprintf(startUrl, common.ConsumerSignalingPort) - req, err := http.NewRequestWithContext(ctx, "POST", consumerSignalingUrl, bytes.NewBuffer(serialized)) + status, err := c.send(ctx, fmt.Sprintf(signalingUrl+"/prepare", common.ConsumerSignalingPort), prepareMessage) if err != nil { - return fmt.Errorf("failed to create request: %w", err) + return nil, fmt.Errorf("prepare request failed: %w", err) } + return status.DataAddress, nil +} - req.Header.Set(contentType, jsonContentType) - - client := &http.Client{ - Timeout: 30 * time.Second, +// ConsumerStarted notifies the consumer data plane that the provider started the transfer. For pull transfers source +// is the provider's DataAddress; for push transfers it must be nil. +func (c *ControlPlaneSimulator) ConsumerStarted(ctx context.Context, dataFlowID string, source *dsdk.DataAddress) error { + message := dsdk.DataFlowStartedNotificationMessage{ + MessageID: uuid.NewString(), + DataAddress: source, } - - resp, err := client.Do(req) + _, err := c.send(ctx, fmt.Sprintf(signalingUrl+"/%s/started", common.ConsumerSignalingPort, dataFlowID), message) if err != nil { - return fmt.Errorf("start request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - var message dsdk.DataFlowResponseMessage - if err := json.NewDecoder(resp.Body).Decode(&message); err != nil { - return fmt.Errorf("failed to decode response: %w", err) + return fmt.Errorf("started request failed: %w", err) } - return nil } -func (c *ControlPlaneSimulator) ConsumerPrepare(ctx context.Context, processID string, agreementID string, datasetID string) (*dsdk.DataAddress, error) { - callbackURL, _ := url.Parse(providerCallbackURL) - prepareMessage := dsdk.DataFlowPrepareMessage{ - DataFlowBaseMessage: dsdk.DataFlowBaseMessage{ - MessageID: uuid.NewString(), - AgreementID: agreementID, - DatasetID: datasetID, - ProcessID: processID, - DataspaceContext: "dscontext", - ParticipantID: "did:web:consumer.com", - CounterPartyID: "did:web:provider.com", - CallbackAddress: dsdk.CallbackURL(*callbackURL), - TransferType: dsdk.TransferType{DestinationType: "custom", FlowType: dsdk.Pull}, - }, - } - - serialized, err := json.Marshal(prepareMessage) - if err != nil { - return nil, fmt.Errorf("failed to marshal prepare message: %w", err) - } - - // Create the request - consumerSignalingUrl := fmt.Sprintf(consumerPrepareURL, common.ConsumerSignalingPort) - req, err := http.NewRequestWithContext(ctx, "POST", consumerSignalingUrl, bytes.NewBuffer(serialized)) +func (c *ControlPlaneSimulator) ProviderTerminate(ctx context.Context, dataFlowID string) error { + terminateMessage := dsdk.DataFlowTerminateMessage{MessageID: uuid.NewString(), Reason: "violation"} + resp, err := c.post(ctx, fmt.Sprintf(signalingUrl+"/%s/terminate", common.ProviderSignalingPort, dataFlowID), terminateMessage) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return fmt.Errorf("terminate request failed: %w", err) } + _ = resp.Body.Close() + return nil +} - req.Header.Set(contentType, jsonContentType) - - client := &http.Client{ - Timeout: 30 * time.Second, +func newBaseMessage(dataFlowID, agreementID, datasetID, profile, participantID, counterPartyID string) dsdk.DataFlowBaseMessage { + return dsdk.DataFlowBaseMessage{ + MessageID: uuid.NewString(), + ParticipantID: participantID, + CounterPartyID: counterPartyID, + DataspaceContext: "dscontext", + DataFlowID: dataFlowID, + AgreementID: agreementID, + DatasetID: datasetID, + Profile: profile, + Claims: map[string]any{"membership": "active"}, } +} - resp, err := client.Do(req) +// send posts the message and decodes the DataFlowStatusMessage of a 200 response. +func (c *ControlPlaneSimulator) send(ctx context.Context, url string, message any) (*dsdk.DataFlowStatusMessage, error) { + resp, err := c.post(ctx, url, message) if err != nil { - return nil, fmt.Errorf("prepare request failed: %w", err) + return nil, err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - var message dsdk.DataFlowResponseMessage - if err := json.NewDecoder(resp.Body).Decode(&message); err != nil { + var status dsdk.DataFlowStatusMessage + if err := json.NewDecoder(resp.Body).Decode(&status); err != nil { return nil, fmt.Errorf("failed to decode response: %w", err) } - - return message.DataAddress, nil + return &status, nil } -func (c *ControlPlaneSimulator) ProviderTerminate(ctx context.Context, processID string, agreementID string, datasetID string) error { - terminateMessage := dsdk.DataFlowTransitionMessage{Reason: "violation"} - - serialized, err := json.Marshal(terminateMessage) +// post sends the message and returns the response if the data plane answered with 200 OK. +func (c *ControlPlaneSimulator) post(ctx context.Context, url string, message any) (*http.Response, error) { + serialized, err := json.Marshal(message) if err != nil { - return fmt.Errorf("failed to marshal terminate message: %w", err) + return nil, fmt.Errorf("failed to marshal message: %w", err) } - // Create the request - providerSignalingUrl := fmt.Sprintf(terminateUrl, common.ProviderSignalingPort, processID) - req, err := http.NewRequestWithContext(ctx, "POST", providerSignalingUrl, bytes.NewBuffer(serialized)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(serialized)) if err != nil { - return fmt.Errorf("failed to create request: %w", err) + return nil, fmt.Errorf("failed to create request: %w", err) } - req.Header.Set(contentType, jsonContentType) - client := &http.Client{ - Timeout: 30 * time.Second, - } - - resp, err := client.Do(req) + resp, err := c.client.Do(req) if err != nil { - return fmt.Errorf("start request failed: %w", err) + return nil, err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, body) } - - return nil + return resp, nil } diff --git a/examples/natsservices/constants.go b/examples/natsservices/constants.go index 91bbf19..7bf5f69 100644 --- a/examples/natsservices/constants.go +++ b/examples/natsservices/constants.go @@ -14,9 +14,13 @@ package natsservices const ( NATSEndpointType = "https://test.org/natsdp/v1/nats" - TokenKey = "token" - ForwardSuffix = "forward" - ReplySuffix = "reply" - ChannelKey = "channel" - ReplyChannelKey = "replyChannel" + // PullProfile and PushProfile are the transfer profiles of the streaming samples; the SDK derives the transfer + // type from the last '-' token. + PullProfile = "https://test.org/natsdp/v1/nats-pull" + PushProfile = "https://test.org/natsdp/v1/nats-push" + TokenKey = "token" + ForwardSuffix = "forward" + ReplySuffix = "reply" + ChannelKey = "channel" + ReplyChannelKey = "replyChannel" ) diff --git a/examples/streaming-pull-dataplane/consumer/dataplane.go b/examples/streaming-pull-dataplane/consumer/dataplane.go index ef25c75..dad6aa4 100644 --- a/examples/streaming-pull-dataplane/consumer/dataplane.go +++ b/examples/streaming-pull-dataplane/consumer/dataplane.go @@ -25,7 +25,7 @@ import ( ) type ConsumerDataPlane struct { - api *dsdk.DataPlaneApi + sdk *dsdk.DataPlaneSDK signalingServer *http.Server dataServer *http.Server eventSubscriber *natsservices.EventSubscriber @@ -43,12 +43,12 @@ func NewDataPlane(eventSubscriber *natsservices.EventSubscriber) (*ConsumerDataP if err != nil { return nil, err } - dataplane.api = dsdk.NewDataPlaneApi(sdk) + dataplane.sdk = sdk return dataplane, nil } func (d *ConsumerDataPlane) Init() { - d.signalingServer = common.NewSignalingServer(d.api, common.ConsumerSignalingPort) + d.signalingServer = common.NewSignalingServer(d.sdk, common.ConsumerSignalingPort) // Start signaling server go func() { log.Printf("[Consumer Data Plane] Signaling server listening on port %d\n", common.ConsumerSignalingPort) @@ -70,18 +70,18 @@ func (d *ConsumerDataPlane) Shutdown(ctx context.Context) { func (d *ConsumerDataPlane) prepareProcessor(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - _ *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { log.Printf("[Consumer Data Plane] Prepared transfer for participant %s dataset %s\n", flow.ParticipantID, flow.DatasetID) - return &dsdk.DataFlowResponseMessage{State: dsdk.Prepared}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared}, nil } func (d *ConsumerDataPlane) startProcessor(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { - endpoint := options.DataAddress.Properties[dsdk.EndpointKey].(string) + endpoint := options.DataAddress.Endpoint token, found := parseToken(natsservices.TokenKey, options.DataAddress) if !found { return nil, errors.New("token not found in endpoint properties") @@ -98,7 +98,7 @@ func (d *ConsumerDataPlane) startProcessor(_ context.Context, return nil, err } log.Printf("[Consumer Data Plane] Started NATS subscriber for participant %s dataset %s\n", flow.ParticipantID, flow.DatasetID) - return &dsdk.DataFlowResponseMessage{State: dsdk.Started}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Started}, nil } func (d *ConsumerDataPlane) terminateProcessor(_ context.Context, flow *dsdk.DataFlow) error { @@ -114,14 +114,5 @@ func (d *ConsumerDataPlane) suspendProcessor(_ context.Context, flow *dsdk.DataF } func parseToken(keyValue string, da *dsdk.DataAddress) (string, bool) { - rawProps, ok := da.Properties[dsdk.EndpointProperties].([]any) - if !ok { - log.Fatal("endpoint properties is not a slice") - } - for _, item := range rawProps { - if item.(map[string]any)["key"] == keyValue { - return item.(map[string]any)["value"].(string), true - } - } - return "", false + return da.Property(keyValue) } diff --git a/examples/streaming-pull-dataplane/main.go b/examples/streaming-pull-dataplane/main.go index d3cd9f0..69c3131 100644 --- a/examples/streaming-pull-dataplane/main.go +++ b/examples/streaming-pull-dataplane/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/eclipse-dataplane-core/dataplane-sdk-go/examples/natsservices" "github.com/eclipse-dataplane-core/dataplane-sdk-go/examples/streaming" "github.com/eclipse-dataplane-core/dataplane-sdk-go/examples/streaming-pull-dataplane/launcher" ) @@ -8,5 +9,5 @@ import ( func main() { launcher.LaunchServices() - streaming.TerminateScenario() + streaming.TerminateScenario(natsservices.PullProfile) } diff --git a/examples/streaming-pull-dataplane/provider/dataplane.go b/examples/streaming-pull-dataplane/provider/dataplane.go index 0db9dca..0d5e293 100644 --- a/examples/streaming-pull-dataplane/provider/dataplane.go +++ b/examples/streaming-pull-dataplane/provider/dataplane.go @@ -27,7 +27,7 @@ import ( // ProviderDataPlane demonstrates how to use the Data Plane SDK. This implementation supports pull event streaming. type ProviderDataPlane struct { - api *dsdk.DataPlaneApi + sdk *dsdk.DataPlaneSDK signalingServer *http.Server authService *natsservices.AuthService connectionInvalidator ConnectionInvalidator @@ -51,13 +51,13 @@ func NewDataPlane(authService *natsservices.AuthService, return nil, err } - providerDataPlane.api = dsdk.NewDataPlaneApi(sdk) + providerDataPlane.sdk = sdk return providerDataPlane, nil } func (d *ProviderDataPlane) Init() { - d.signalingServer = common.NewSignalingServer(d.api, common.ProviderSignalingPort) + d.signalingServer = common.NewSignalingServer(d.sdk, common.ProviderSignalingPort) // Start signaling server go func() { @@ -80,14 +80,14 @@ func (d *ProviderDataPlane) Shutdown(ctx context.Context) { func (d *ProviderDataPlane) prepareProcessor(_ context.Context, _ *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - _ *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { return nil, errors.New("not supported on provider") } func (d *ProviderDataPlane) startProcessor(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { if options.Duplicate { // Perform de-duplication. This code path is not needed, but it demonstrates how de-deduplication can be handled } @@ -100,11 +100,11 @@ func (d *ProviderDataPlane) startProcessor(_ context.Context, channel := flow.ID + "." + natsservices.ForwardSuffix replyChannel := flow.ID + "." + natsservices.ReplySuffix da, err := dsdk.NewDataAddressBuilder(). - Property(dsdk.EndpointType, natsservices.NATSEndpointType). - Property(dsdk.EndpointKey, natsservices.NatsUrl). - EndpointProperty(natsservices.TokenKey, "string", token). - EndpointProperty(natsservices.ChannelKey, "string", channel). - EndpointProperty(natsservices.ReplyChannelKey, "string", replyChannel). + EndpointType(natsservices.NATSEndpointType). + Endpoint(natsservices.NatsUrl). + Property(natsservices.TokenKey, token). + Property(natsservices.ChannelKey, channel). + Property(natsservices.ReplyChannelKey, replyChannel). Build() if err != nil { return nil, fmt.Errorf("failed to build data address: %w", err) @@ -114,7 +114,7 @@ func (d *ProviderDataPlane) startProcessor(_ context.Context, d.publisherService.Start(channel) log.Printf("[Provider Data Plane] Starting transfer for %s\n", flow.CounterPartyID) - return &dsdk.DataFlowResponseMessage{State: dsdk.Started, DataAddress: da}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Started, DataAddress: da}, nil } func (d *ProviderDataPlane) suspendProcessor(_ context.Context, flow *dsdk.DataFlow) error { diff --git a/examples/streaming-pull-dataplane/streaming-pull-dataplane b/examples/streaming-pull-dataplane/streaming-pull-dataplane new file mode 100755 index 0000000..f757e38 Binary files /dev/null and b/examples/streaming-pull-dataplane/streaming-pull-dataplane differ diff --git a/examples/streaming-push-dataplane/consumer/dataplane.go b/examples/streaming-push-dataplane/consumer/dataplane.go index 865c5da..dbc7bb3 100644 --- a/examples/streaming-push-dataplane/consumer/dataplane.go +++ b/examples/streaming-push-dataplane/consumer/dataplane.go @@ -27,7 +27,7 @@ import ( // ConsumerDataPlane demonstrates how to use the Data Plane SDK. This implementation supports push event streaming. type ConsumerDataPlane struct { - api *dsdk.DataPlaneApi + sdk *dsdk.DataPlaneSDK signalingServer *http.Server authService *natsservices.AuthService connectionInvalidator ConnectionInvalidator @@ -58,13 +58,13 @@ func NewDataPlane(authService *natsservices.AuthService, return nil, err } - dataPlane.api = dsdk.NewDataPlaneApi(sdk) + dataPlane.sdk = sdk return dataPlane, nil } func (d *ConsumerDataPlane) Init() { - d.signalingServer = common.NewSignalingServer(d.api, common.ConsumerSignalingPort) + d.signalingServer = common.NewSignalingServer(d.sdk, common.ConsumerSignalingPort) // Start signaling server go func() { @@ -87,7 +87,7 @@ func (d *ConsumerDataPlane) Shutdown(ctx context.Context) { func (d *ConsumerDataPlane) prepareProcessor(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { if options.Duplicate { // Perform de-duplication. This code path is not needed, but it demonstrates how de-deduplication can be handled } @@ -106,25 +106,25 @@ func (d *ConsumerDataPlane) prepareProcessor(_ context.Context, replyChannel := flow.ID + "." + natsservices.ReplySuffix da, err := dsdk.NewDataAddressBuilder(). - Property(dsdk.EndpointType, natsservices.NATSEndpointType). - Property(dsdk.EndpointKey, natsservices.NatsUrl). - EndpointProperty(natsservices.TokenKey, "string", token). - EndpointProperty(natsservices.ChannelKey, "string", channel). - EndpointProperty(natsservices.ReplyChannelKey, "string", replyChannel). + EndpointType(natsservices.NATSEndpointType). + Endpoint(natsservices.NatsUrl). + Property(natsservices.TokenKey, token). + Property(natsservices.ChannelKey, channel). + Property(natsservices.ReplyChannelKey, replyChannel). Build() if err != nil { return nil, fmt.Errorf("failed to build data address: %w", err) } log.Printf("[Consumer Data Plane] Prepare transfer for %s\n", flow.CounterPartyID) - return &dsdk.DataFlowResponseMessage{State: dsdk.Prepared, DataAddress: da}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared, DataAddress: da}, nil } func (d *ConsumerDataPlane) startProcessor(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { - return &dsdk.DataFlowResponseMessage{State: dsdk.Started, DataAddress: options.DataAddress}, nil + options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { + return &dsdk.DataFlowStatusMessage{State: dsdk.Started, DataAddress: options.DataAddress}, nil } func (d *ConsumerDataPlane) suspendProcessor(_ context.Context, flow *dsdk.DataFlow) error { diff --git a/examples/streaming-push-dataplane/main.go b/examples/streaming-push-dataplane/main.go index fdc7d86..6c64df2 100644 --- a/examples/streaming-push-dataplane/main.go +++ b/examples/streaming-push-dataplane/main.go @@ -13,6 +13,7 @@ package main import ( + "github.com/eclipse-dataplane-core/dataplane-sdk-go/examples/natsservices" "github.com/eclipse-dataplane-core/dataplane-sdk-go/examples/streaming" "github.com/eclipse-dataplane-core/dataplane-sdk-go/examples/streaming-push-dataplane/launcher" ) @@ -20,5 +21,5 @@ import ( func main() { launcher.LaunchServices() - streaming.TerminateScenario() + streaming.TerminateScenario(natsservices.PushProfile) } diff --git a/examples/streaming-push-dataplane/provider/dataplane.go b/examples/streaming-push-dataplane/provider/dataplane.go index 46a5f71..1af3421 100644 --- a/examples/streaming-push-dataplane/provider/dataplane.go +++ b/examples/streaming-push-dataplane/provider/dataplane.go @@ -25,7 +25,7 @@ import ( ) type ProviderDataPlane struct { - api *dsdk.DataPlaneApi + sdk *dsdk.DataPlaneSDK signalingServer *http.Server dataServer *http.Server publisherService *EventPublisherService @@ -45,12 +45,12 @@ func NewDataPlane(publisherService *EventPublisherService) (*ProviderDataPlane, if err != nil { return nil, err } - dataplane.api = dsdk.NewDataPlaneApi(sdk) + dataplane.sdk = sdk return dataplane, nil } func (d *ProviderDataPlane) Init() { - d.signalingServer = common.NewSignalingServer(d.api, common.ProviderSignalingPort) + d.signalingServer = common.NewSignalingServer(d.sdk, common.ProviderSignalingPort) // Start signaling server go func() { log.Printf("[Provider Data Plane] Signaling server listening on port %d\n", common.ProviderSignalingPort) @@ -72,16 +72,16 @@ func (d *ProviderDataPlane) Shutdown(ctx context.Context) { func (d *ProviderDataPlane) prepareProcessor(_ context.Context, _ *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - _ *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { return nil, errors.New("not supported on provider") } func (d *ProviderDataPlane) startProcessor(ctx context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { - endpoint := options.DataAddress.Properties[dsdk.EndpointKey].(string) + endpoint := options.DataAddress.Endpoint token, found := parseToken(natsservices.TokenKey, options.DataAddress) if !found { return nil, errors.New("token not found in endpoint properties") @@ -96,7 +96,7 @@ func (d *ProviderDataPlane) startProcessor(ctx context.Context, d.publisherService.Start(flow.ID, endpoint, channel, token) log.Printf("[Provider Data Plane] Started NATS subscriber for participant %s dataset %s\n", flow.ParticipantID, flow.DatasetID) - return &dsdk.DataFlowResponseMessage{State: dsdk.Started}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Started}, nil } func (d *ProviderDataPlane) terminateProcessor(_ context.Context, flow *dsdk.DataFlow) error { @@ -114,14 +114,5 @@ func (d *ProviderDataPlane) suspendProcessor(_ context.Context, flow *dsdk.DataF } func parseToken(keyValue string, da *dsdk.DataAddress) (string, bool) { - rawProps, ok := da.Properties[dsdk.EndpointProperties].([]any) - if !ok { - log.Fatal("endpoint properties is not a slice") - } - for _, item := range rawProps { - if item.(map[string]any)["key"] == keyValue { - return item.(map[string]any)["value"].(string), true - } - } - return "", false + return da.Property(keyValue) } diff --git a/examples/streaming/terminate.go b/examples/streaming/terminate.go index cb32534..b9b09f2 100644 --- a/examples/streaming/terminate.go +++ b/examples/streaming/terminate.go @@ -22,7 +22,8 @@ import ( ) // TerminateScenario coordinates a simulated data transfer scenario and forcibly terminates it after a predefined duration. -func TerminateScenario() { +// TerminateScenario runs a streaming transfer with the given profile (ending in -push or -pull) and terminates it. +func TerminateScenario(profile string) { cp, err := controlplane.NewSimulator() if err != nil { log.Fatalf("Unable to launch control plane simulator: %v\n", err) @@ -35,28 +36,28 @@ func TerminateScenario() { datasetID := uuid.NewString() consumerProcessID := uuid.NewString() - consumerDA, err := cp.ConsumerPrepare(ctx, consumerProcessID, agreementID, datasetID) + consumerDA, err := cp.ConsumerPrepare(ctx, consumerProcessID, agreementID, datasetID, profile) if err != nil { log.Fatalf("Unable to send prepare to consumer control plane: %v\n", err) } providerProcessID := uuid.NewString() // Signal to the provider data plane - providerDA, err := cp.ProviderStart(ctx, providerProcessID, agreementID, datasetID, consumerDA) + providerDA, err := cp.ProviderStart(ctx, providerProcessID, agreementID, datasetID, profile, consumerDA) if err != nil { log.Fatalf("Unable to send start to provider control plane: %v\n", err) } - err = cp.ConsumerStart(ctx, consumerProcessID, providerDA) + err = cp.ConsumerStarted(ctx, consumerProcessID, providerDA) if err != nil { - log.Fatalf("Unable to send start to consumer control plane: %v\n", err) + log.Fatalf("Unable to send started to consumer control plane: %v\n", err) } // Simulate ongoing processing time.Sleep(10 * time.Second) // Terminate the transfer and forcibly disconnect the client - err = cp.ProviderTerminate(ctx, providerProcessID, agreementID, datasetID) + err = cp.ProviderTerminate(ctx, providerProcessID) if err != nil { log.Fatalf("Unable to send terminate to provider control plane: %v\n", err) } diff --git a/examples/sync-pull-dataplane/consumer/dataplane.go b/examples/sync-pull-dataplane/consumer/dataplane.go index dd17194..e03ad81 100644 --- a/examples/sync-pull-dataplane/consumer/dataplane.go +++ b/examples/sync-pull-dataplane/consumer/dataplane.go @@ -39,7 +39,7 @@ const ( // This data plane implements non-finite data transfers. Multiple requests may be issued to the provider data plane over a // period of time. For example, the dataset could be access to an API. type ConsumerDataPlane struct { - api *dsdk.DataPlaneApi + sdk *dsdk.DataPlaneSDK signalingServer *http.Server dataServer *http.Server tokenStore *common.Store[tokenEntry] @@ -57,12 +57,12 @@ func NewDataPlane() (*ConsumerDataPlane, error) { if err != nil { return nil, err } - dataplane.api = dsdk.NewDataPlaneApi(sdk) + dataplane.sdk = sdk return dataplane, nil } func (d *ConsumerDataPlane) Init() { - d.signalingServer = common.NewSignalingServer(d.api, common.ConsumerSignalingPort) + d.signalingServer = common.NewSignalingServer(d.sdk, common.ConsumerSignalingPort) // Start signaling server go func() { log.Printf("[Consumer Data Plane] Signaling server listening on port %d\n", common.ConsumerSignalingPort) @@ -92,20 +92,23 @@ func (d *ConsumerDataPlane) Shutdown(ctx context.Context) { func (d *ConsumerDataPlane) prepareProcessor(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - _ *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { log.Printf("[Consumer Data Plane] Prepared transfer for participant %s dataset %s\n", flow.ParticipantID, flow.DatasetID) - return &dsdk.DataFlowResponseMessage{State: dsdk.Prepared}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared}, nil } func (d *ConsumerDataPlane) startProcessor(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, - options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { + options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { log.Printf("[Consumer Data Plane] Transfer access token available for participant %s dataset %s\n", flow.ParticipantID, flow.DatasetID) - endpoint := options.DataAddress.Properties[dsdk.EndpointKey].(string) - token := options.DataAddress.Properties["token"].(string) + endpoint := options.DataAddress.Endpoint + token, found := options.DataAddress.Property("token") + if !found { + return nil, errors.New("token not found in endpoint properties") + } d.tokenStore.Create(flow.DatasetID, tokenEntry{datasetID: flow.DatasetID, token: token, endpoint: endpoint}) - return &dsdk.DataFlowResponseMessage{State: dsdk.Started}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Started}, nil } func (d *ConsumerDataPlane) getEndpointToken(w http.ResponseWriter, r *http.Request) { diff --git a/examples/sync-pull-dataplane/main.go b/examples/sync-pull-dataplane/main.go index b2cf3d2..2cb3561 100644 --- a/examples/sync-pull-dataplane/main.go +++ b/examples/sync-pull-dataplane/main.go @@ -23,6 +23,8 @@ import ( "github.com/google/uuid" ) +const pullProfile = "https://w3id.org/dspace-sig/profile/http-pull" + // Demonstrates initiating a data transfer using a provider data plane that implements synchronous signaling start operations. func main() { @@ -44,22 +46,22 @@ func main() { datasetID := uuid.NewString() consumerProcessID := uuid.NewString() - _, err = cp.ConsumerPrepare(ctx, consumerProcessID, agreementID, datasetID) + _, err = cp.ConsumerPrepare(ctx, consumerProcessID, agreementID, datasetID, pullProfile) if err != nil { log.Fatalf("Unable to send prepare to consumer control plane: %v\n", err) } // Signal to the provider data plane providerProcessID := uuid.NewString() - da, err := cp.ProviderStart(ctx, providerProcessID, agreementID, datasetID, nil) + da, err := cp.ProviderStart(ctx, providerProcessID, agreementID, datasetID, pullProfile, nil) if err != nil { log.Fatalf("Unable to send start to provider control plane: %v\n", err) } // Signal to the consumer data plane in response - err = cp.ConsumerStart(ctx, consumerProcessID, da) + err = cp.ConsumerStarted(ctx, consumerProcessID, da) if err != nil { - log.Fatalf("Unable to send start to consumer control plane: %v\n", err) + log.Fatalf("Unable to send started to consumer control plane: %v\n", err) } if err = consumer.TransferDataset(datasetID); err != nil { diff --git a/examples/sync-pull-dataplane/provider/dataplane.go b/examples/sync-pull-dataplane/provider/dataplane.go index 1a1fcfd..d840a3c 100644 --- a/examples/sync-pull-dataplane/provider/dataplane.go +++ b/examples/sync-pull-dataplane/provider/dataplane.go @@ -27,15 +27,16 @@ import ( ) const ( - contentType = "Content-Type" - jsonContentType = "application/json" - endpointUrl = "http://localhost:%d/datasets" + contentType = "Content-Type" + jsonContentType = "application/json" + endpointUrl = "http://localhost:%d/datasets" + httpEndpointType = "https://w3id.org/idsa/v4.1/HTTP" ) // ProviderDataPlane is a provider data plane that demonstrates how to use the Data Plane SDK. This implementation supports // the transfer of simple JSON datasets over HTTP and Data Plane Signaling start and prepare handling using synchronous responses. type ProviderDataPlane struct { - api *dsdk.DataPlaneApi + sdk *dsdk.DataPlaneSDK tokenStore *common.Store[tokenEntry] signalingServer *http.Server dataServer *http.Server @@ -58,13 +59,13 @@ func NewDataPlane() (*ProviderDataPlane, error) { return nil, err } - providerDataPlane.api = dsdk.NewDataPlaneApi(sdk) + providerDataPlane.sdk = sdk return providerDataPlane, nil } func (d *ProviderDataPlane) Init() { - d.signalingServer = common.NewSignalingServer(d.api, common.ProviderSignalingPort) + d.signalingServer = common.NewSignalingServer(d.sdk, common.ProviderSignalingPort) d.dataServer = common.NewDataServer(common.ProviderDataPort, "/datasets/", d.transferDataset) // Start signaling server @@ -98,11 +99,11 @@ func (d *ProviderDataPlane) Shutdown(ctx context.Context) { log.Println("Provider data plane shutdown") } -func (d *ProviderDataPlane) prepareProcessor(ctx context.Context, flow *dsdk.DataFlow, sdk *dsdk.DataPlaneSDK, options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { +func (d *ProviderDataPlane) prepareProcessor(ctx context.Context, flow *dsdk.DataFlow, sdk *dsdk.DataPlaneSDK, options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { return nil, errors.New("not supported on provider") } -func (d *ProviderDataPlane) startProcessor(ctx context.Context, flow *dsdk.DataFlow, sdk *dsdk.DataPlaneSDK, options *dsdk.ProcessorOptions) (*dsdk.DataFlowResponseMessage, error) { +func (d *ProviderDataPlane) startProcessor(ctx context.Context, flow *dsdk.DataFlow, sdk *dsdk.DataPlaneSDK, options *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { token := uuid.NewString() if options.Duplicate { @@ -119,8 +120,9 @@ func (d *ProviderDataPlane) startProcessor(ctx context.Context, flow *dsdk.DataF d.tokenStore.Create(flow.DatasetID, tokenEntry) // token is pinned to the flow ID which is the transfer process id on the control plane da, err := dsdk.NewDataAddressBuilder(). + EndpointType(httpEndpointType). + Endpoint(fmt.Sprintf(endpointUrl, common.ProviderDataPort)). Property("token", token). - Property("endpoint", fmt.Sprintf(endpointUrl, common.ProviderDataPort)). Build() if err != nil { // remove up token on error @@ -129,7 +131,7 @@ func (d *ProviderDataPlane) startProcessor(ctx context.Context, flow *dsdk.DataF } log.Printf("[Provider Data Plane] Started transfer for %s and returning access token\n", flow.CounterPartyID) - return &dsdk.DataFlowResponseMessage{State: dsdk.Started, DataAddress: da}, nil + return &dsdk.DataFlowStatusMessage{State: dsdk.Started, DataAddress: da}, nil } func (d *ProviderDataPlane) suspendProcessor(ctx context.Context, flow *dsdk.DataFlow) error { diff --git a/go.mod b/go.mod index a911566..93a6464 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/eclipse-dataplane-core/dataplane-sdk-go go 1.24.1 require ( - github.com/go-chi/chi/v5 v5.5.1-0.20261018135022-9f8adb1aabaa + github.com/docker/go-connections v0.6.0 github.com/go-playground/validator/v10 v10.27.0 github.com/google/uuid v1.6.0 github.com/lib/pq v1.10.9 @@ -11,6 +11,7 @@ require ( github.com/nats-io/nats-server/v2 v2.11.6 github.com/nats-io/nats.go v1.43.0 github.com/nats-io/nkeys v0.4.11 + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 github.com/stretchr/testify v1.10.0 github.com/synadia-io/callout.go v0.2.1 github.com/testcontainers/testcontainers-go v0.39.0 @@ -29,7 +30,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v28.3.3+incompatible // indirect - github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect diff --git a/go.sum b/go.sum index 3f4317c..2fc90f3 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= @@ -42,10 +44,6 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= -github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= -github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= -github.com/go-chi/chi/v5 v5.5.1-0.20261018135022-9f8adb1aabaa h1:bAIp0rpGqGbS/4/TAZVXIxsNu295q0ZZWh28VSn4EXA= -github.com/go-chi/chi/v5 v5.5.1-0.20261018135022-9f8adb1aabaa/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -135,6 +133,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= diff --git a/internal/tests/api_integration_test.go b/internal/tests/api_integration_test.go index aafa0fc..a82a61e 100644 --- a/internal/tests/api_integration_test.go +++ b/internal/tests/api_integration_test.go @@ -16,537 +16,588 @@ import ( "github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/dsdk" "github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/postgres" - "github.com/go-chi/chi/v5" "github.com/google/uuid" _ "github.com/lib/pq" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + pullProfile = "https://w3id.org/dspace-sig/profile/http-pull" + pushProfile = "https://w3id.org/dspace-sig/profile/s3-push" + // asyncKey in the message metadata makes the test processors report PREPARING/STARTING + asyncKey = "async" ) -// TestMain ensures containers are cleaned up and allows global setup if desired. var ctx = context.Background() -// database SQL database connection, used for test setup and assertionss +// database SQL database connection, used for test setup and assertions var database *sql.DB -// newServerWithSdk instantiates a new HTTP server using the DataPlane SDK and registers its callbacks with endpoints -func newServerWithSdk(t *testing.T, sdk *dsdk.DataPlaneSDK) http.Handler { - t.Helper() - sdkApi := dsdk.NewDataPlaneApi(sdk) - r := chi.NewRouter() - - r.Post("/dataflows/start", sdkApi.Start) - r.Post("/dataflows/{id}/started", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.StartById(writer, request, id) - }) - r.Post("/dataflows/prepare", sdkApi.Prepare) - r.Post("/dataflows/{id}/terminate", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.Terminate(id, writer, request) - }) - r.Post("/dataflows/{id}/suspend", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.Suspend(id, writer, request) - }) - r.Get("/dataflows/{id}/status", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.Status(id, writer, request) - }) - - r.Post("/dataflows/{id}/completed", func(writer http.ResponseWriter, request *http.Request) { - id := chi.URLParam(request, "id") - sdkApi.Complete(id, writer, request) - }) - return r -} - var handler http.Handler func TestMain(m *testing.M) { db, container := postgres.SetupDatabase(&testing.T{}, ctx) database = db - t := &testing.T{} sdk, err := newSdk(db) - assert.NoError(t, err) - handler = newServerWithSdk(t, sdk) + if err != nil { + panic(err) + } + handler = dsdk.NewHandler(sdk) code := m.Run() _ = db.Close() _ = container.Terminate(ctx) os.Exit(code) } -// E2E tests -func Test_Start_NotYetExists(t *testing.T) { +// --- prepare --- - payload, err := serialize(newStartMessage()) - assert.NoError(t, err) +func Test_Prepare_Pull(t *testing.T) { + msg := newPrepareMessage(pullProfile) + rr := post(t, "/v1/dataflows/prepare", msg) - req, err := http.NewRequest(http.MethodPost, "/dataflows/start", bytes.NewBuffer(payload)) - assert.NoError(t, err) + assert.Equal(t, http.StatusOK, rr.Code) + status := decodeStatus(t, rr) + assert.Equal(t, dsdk.Prepared, status.State) + assert.Equal(t, msg.DataFlowID, status.DataFlowID) + assert.NotEmpty(t, status.MessageID) + assert.Nil(t, status.DataAddress) + assert.Equal(t, dsdk.Prepared, findFlow(t, msg.DataFlowID).State) +} - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Prepare_Push_ReturnsDataAddress(t *testing.T) { + msg := newPrepareMessage(pushProfile) + rr := post(t, "/v1/dataflows/prepare", msg) assert.Equal(t, http.StatusOK, rr.Code) - - var responseMessage dsdk.DataFlowResponseMessage - err = json.NewDecoder(rr.Body).Decode(&responseMessage) - assert.NoError(t, err) - assert.Equal(t, responseMessage.State, dsdk.Started) + status := decodeStatus(t, rr) + require.NotNil(t, status.DataAddress) + assert.Equal(t, "https://consumer.example.com/push", status.DataAddress.Endpoint) } -func Test_Start_InvalidPayload(t *testing.T) { - sm := newStartMessage() - sm.CounterPartyID = "" // should raise a validation error - payload, err := serialize(sm) - assert.NoError(t, err) - req, err := http.NewRequest(http.MethodPost, "/dataflows/start", bytes.NewBuffer(payload)) - assert.NoError(t, err) +func Test_Prepare_Async(t *testing.T) { + msg := newPrepareMessage(pullProfile) + msg.Metadata = map[string]any{asyncKey: true} + rr := post(t, "/v1/dataflows/prepare", msg) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) - assert.Equal(t, http.StatusBadRequest, rr.Code) - assert.NotNil(t, rr.Body.String()) + assert.Equal(t, http.StatusAccepted, rr.Code) + assert.Equal(t, "/v1/dataflows/"+msg.DataFlowID+"/status", rr.Header().Get("Location")) + assert.Equal(t, dsdk.Preparing, decodeStatus(t, rr).State) } -func Test_StartByID_WhenNotFound(t *testing.T) { - id := uuid.New().String() - - requestBody, err := serialize(newStartByIdMessage()) - assert.NoError(t, err) +func Test_Prepare_InvalidPayload(t *testing.T) { + pm := newPrepareMessage(pullProfile) + pm.CounterPartyID = "" // should raise a validation error + rr := post(t, "/v1/dataflows/prepare", pm) + assert.Equal(t, http.StatusBadRequest, rr.Code) - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+id+"/started", bytes.NewBuffer(requestBody)) - assert.NoError(t, err) + // the handler must stop after the validation error and write exactly one response body + decoder := json.NewDecoder(rr.Body) + var responseMessage dsdk.ErrorResponse + assert.NoError(t, decoder.Decode(&responseMessage)) + assert.NotEmpty(t, responseMessage.Error) + assert.False(t, decoder.More()) +} +func Test_Prepare_MalformedJSON(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "/v1/dataflows/prepare", strings.NewReader("{")) + require.NoError(t, err) rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusNotFound, rr.Code) - + assert.Equal(t, http.StatusBadRequest, rr.Code) } -func Test_StartByID_WhenStartedOrStarting(t *testing.T) { - - states := []dsdk.DataFlowState{ - dsdk.Started, - dsdk.Starting, - } +func Test_Prepare_WrongState(t *testing.T) { + msg := newPrepareMessage(pullProfile) + createFlow(t, msg.DataFlowID, pullProfile, dsdk.Started, true) - for _, state := range states { - id := uuid.New().String() - store := postgres.NewStore(database) - flow, err := newFlowBuilder().ID(id).State(state).Consumer(true).Build() - assert.NoError(t, err) - assert.NoError(t, store.Create(ctx, flow)) + rr := post(t, "/v1/dataflows/prepare", msg) + assert.Equal(t, http.StatusConflict, rr.Code) +} - requestBody, err := serialize(newStartByIdMessage()) - assert.NoError(t, err) +// --- start --- - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+id+"/started", bytes.NewBuffer(requestBody)) - assert.NoError(t, err) +func Test_Start_Pull_ReturnsDataAddress(t *testing.T) { + msg := newStartMessage(pullProfile, nil) + rr := post(t, "/v1/dataflows/start", msg) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusOK, rr.Code) + status := decodeStatus(t, rr) + assert.Equal(t, dsdk.Started, status.State) + assert.Equal(t, msg.DataFlowID, status.DataFlowID) + require.NotNil(t, status.DataAddress) + token, ok := status.DataAddress.Property("authorization") + assert.True(t, ok) + assert.Equal(t, "token-"+msg.DataFlowID, token) + + flow := findFlow(t, msg.DataFlowID) + assert.Equal(t, dsdk.Started, flow.State) + assert.False(t, flow.Consumer) + assert.Equal(t, msg.Claims, flow.Claims) +} - assert.Equal(t, http.StatusOK, rr.Code) - found, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.Equal(t, dsdk.Started, found.State) - } +func Test_Start_Push(t *testing.T) { + msg := newStartMessage(pushProfile, newDataAddress("https://consumer.example.com/push")) + rr := post(t, "/v1/dataflows/start", msg) + assert.Equal(t, http.StatusOK, rr.Code) + assert.Nil(t, decodeStatus(t, rr).DataAddress) + assert.Equal(t, "https://consumer.example.com/push", findFlow(t, msg.DataFlowID).DestinationDataAddress.Endpoint) } -func Test_StartByID_WhenPrepared(t *testing.T) { - - tests := []struct { - isConsumer bool - expectedHttpCode int - expectedState dsdk.DataFlowState - }{ - { - isConsumer: true, - expectedHttpCode: http.StatusOK, - expectedState: dsdk.Started, - }, - { - isConsumer: false, - expectedHttpCode: http.StatusBadRequest, - expectedState: dsdk.Prepared, - }, - } +func Test_Start_Async(t *testing.T) { + msg := newStartMessage(pullProfile, nil) + msg.Metadata = map[string]any{asyncKey: true} + rr := post(t, "/v1/dataflows/start", msg) - for _, test := range tests { - id := uuid.New().String() - store := postgres.NewStore(database) - flow, err := newFlowBuilder().ID(id).State(dsdk.Prepared).Consumer(test.isConsumer).Build() - assert.NoError(t, err) - assert.NoError(t, store.Create(ctx, flow)) + assert.Equal(t, http.StatusAccepted, rr.Code) + assert.Equal(t, "/v1/dataflows/"+msg.DataFlowID+"/status", rr.Header().Get("Location")) + assert.Equal(t, dsdk.Starting, decodeStatus(t, rr).State) +} - requestBody, err := serialize(newStartByIdMessage()) - assert.NoError(t, err) +func Test_Start_PullWithDataAddress(t *testing.T) { + msg := newStartMessage(pullProfile, newDataAddress("https://example.com")) + rr := post(t, "/v1/dataflows/start", msg) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assertNoFlow(t, msg.DataFlowID) +} - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+id+"/started", bytes.NewBuffer(requestBody)) - assert.NoError(t, err) +func Test_Start_PushWithoutDataAddress(t *testing.T) { + msg := newStartMessage(pushProfile, nil) + rr := post(t, "/v1/dataflows/start", msg) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assertNoFlow(t, msg.DataFlowID) +} - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Start_Duplicate(t *testing.T) { + msg := newStartMessage(pullProfile, nil) + assert.Equal(t, http.StatusOK, post(t, "/v1/dataflows/start", msg).Code) - assert.Equal(t, test.expectedHttpCode, rr.Code) - found, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.Equal(t, test.expectedState, found.State) - } + rr := post(t, "/v1/dataflows/start", msg) + assert.Equal(t, http.StatusConflict, rr.Code) +} +func Test_Start_InvalidPayload(t *testing.T) { + sm := newStartMessage(pullProfile, nil) + sm.CounterPartyID = "" // should raise a validation error + rr := post(t, "/v1/dataflows/start", sm) + assert.Equal(t, http.StatusBadRequest, rr.Code) + var errorResponse dsdk.ErrorResponse + assert.NoError(t, json.NewDecoder(rr.Body).Decode(&errorResponse)) + assert.NotEmpty(t, errorResponse.Error) } -func Test_StartByID_MissingSourceAddress(t *testing.T) { - id := uuid.New().String() - store := postgres.NewStore(database) - flow, err := newFlowBuilder().ID(id).State(dsdk.Prepared).Consumer(true).Build() - assert.NoError(t, err) - assert.NoError(t, store.Create(ctx, flow)) - requestBody, err := serialize(dsdk.DataFlowStartedNotificationMessage{}) - assert.NoError(t, err) +// --- started --- - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/started", bytes.NewBuffer(requestBody)) - assert.NoError(t, err) +func Test_Started_Pull(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Prepared, true) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) + rr := post(t, "/v1/dataflows/"+id+"/started", dsdk.DataFlowStartedNotificationMessage{ + MessageID: uuid.NewString(), + DataAddress: newDataAddress("https://provider.example.com/data"), + }) assert.Equal(t, http.StatusOK, rr.Code) + status := decodeStatus(t, rr) + assert.Equal(t, dsdk.Started, status.State) + assert.Equal(t, id, status.DataFlowID) + + flow := findFlow(t, id) + assert.Equal(t, dsdk.Started, flow.State) + assert.Equal(t, "https://provider.example.com/data", flow.SourceDataAddress.Endpoint) } -func Test_Prepare(t *testing.T) { - payload, err := serialize(newPrepareMessage()) - assert.NoError(t, err) +func Test_Started_Push(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pushProfile, dsdk.Prepared, true) - req, err := http.NewRequest(http.MethodPost, "/dataflows/prepare", bytes.NewBuffer(payload)) - assert.NoError(t, err) + rr := post(t, "/v1/dataflows/"+id+"/started", dsdk.DataFlowStartedNotificationMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusOK, rr.Code) + assert.Equal(t, dsdk.Started, decodeStatus(t, rr).State) +} - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Started_Duplicate(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pushProfile, dsdk.Started, true) + rr := post(t, "/v1/dataflows/"+id+"/started", dsdk.DataFlowStartedNotificationMessage{MessageID: uuid.NewString()}) assert.Equal(t, http.StatusOK, rr.Code) + assert.Equal(t, dsdk.Started, decodeStatus(t, rr).State) +} - var responseMessage dsdk.DataFlowResponseMessage - err = json.NewDecoder(rr.Body).Decode(&responseMessage) - assert.NoError(t, err) - assert.Equal(t, responseMessage.State, dsdk.Prepared) +func Test_Started_DataAddressRules(t *testing.T) { + pull := uuid.NewString() + createFlow(t, pull, pullProfile, dsdk.Prepared, true) + rr := post(t, "/v1/dataflows/"+pull+"/started", dsdk.DataFlowStartedNotificationMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusBadRequest, rr.Code) + + push := uuid.NewString() + createFlow(t, push, pushProfile, dsdk.Prepared, true) + rr = post(t, "/v1/dataflows/"+push+"/started", dsdk.DataFlowStartedNotificationMessage{ + MessageID: uuid.NewString(), + DataAddress: newDataAddress("https://example.com"), + }) + assert.Equal(t, http.StatusBadRequest, rr.Code) } -func Test_Prepare_InvalidPayload(t *testing.T) { - pm := newPrepareMessage() - pm.CounterPartyID = "" // should raise a validation error - payload, err := serialize(pm) - assert.NoError(t, err) - req, err := http.NewRequest(http.MethodPost, "/dataflows/prepare", bytes.NewBuffer(payload)) - assert.NoError(t, err) +func Test_Started_MissingMessageID(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pushProfile, dsdk.Prepared, true) + rr := post(t, "/v1/dataflows/"+id+"/started", map[string]any{}) + assert.Equal(t, http.StatusBadRequest, rr.Code) +} - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Started_WrongState(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pushProfile, dsdk.Terminated, true) + rr := post(t, "/v1/dataflows/"+id+"/started", dsdk.DataFlowStartedNotificationMessage{MessageID: uuid.NewString()}) assert.Equal(t, http.StatusBadRequest, rr.Code) +} - // the handler must stop after the validation error and write exactly one response body - decoder := json.NewDecoder(rr.Body) - var responseMessage dsdk.DataFlowResponseMessage - assert.NoError(t, decoder.Decode(&responseMessage)) - assert.NotEmpty(t, responseMessage.Error) - assert.False(t, decoder.More()) +func Test_Started_WhenNotFound(t *testing.T) { + rr := post(t, "/v1/dataflows/"+uuid.NewString()+"/started", dsdk.DataFlowStartedNotificationMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusNotFound, rr.Code) } -func Test_Prepare_WrongState(t *testing.T) { +// --- suspend / resume --- - s := postgres.NewStore(database) - flow, err := newFlowBuilder().State(dsdk.Started).Build() - assert.NoError(t, err) - assert.NoError(t, s.Create(ctx, flow)) +func Test_SuspendAndResume(t *testing.T) { + msg := newStartMessage(pullProfile, nil) + require.Equal(t, http.StatusOK, post(t, "/v1/dataflows/start", msg).Code) + id := msg.DataFlowID - message := newPrepareMessage() - message.ProcessID = flow.ID - payload, err := serialize(message) - assert.NoError(t, err) + rr := post(t, "/v1/dataflows/"+id+"/suspend", dsdk.DataFlowSuspendMessage{MessageID: uuid.NewString(), Reason: "maintenance"}) + assert.Equal(t, http.StatusOK, rr.Code) + flow := findFlow(t, id) + assert.Equal(t, dsdk.Suspended, flow.State) + assert.Equal(t, dsdk.SuspendedByRemote, flow.SuspendedBy) + assert.Equal(t, "maintenance", flow.ErrorDetail) - req, err := http.NewRequest(http.MethodPost, "/dataflows/prepare", bytes.NewBuffer(payload)) - assert.NoError(t, err) + rr = post(t, "/v1/dataflows/"+id+"/resume", dsdk.DataFlowResumeMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusOK, rr.Code) + status := decodeStatus(t, rr) + assert.Equal(t, dsdk.Started, status.State) + assert.Equal(t, id, status.DataFlowID) + require.NotNil(t, status.DataAddress, "a pull provider returns its data address on resume") + assert.Equal(t, "https://provider.example.com/data", status.DataAddress.Endpoint) + + flow = findFlow(t, id) + assert.Equal(t, dsdk.Started, flow.State) + assert.Equal(t, dsdk.SuspendedByNone, flow.SuspendedBy) +} - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Suspend_RequiresBody(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Started, false) - assert.Equal(t, http.StatusConflict, rr.Code) + assert.Equal(t, http.StatusBadRequest, postRaw(t, "/v1/dataflows/"+id+"/suspend", "").Code) + assert.Equal(t, http.StatusBadRequest, post(t, "/v1/dataflows/"+id+"/suspend", map[string]any{"reason": "x"}).Code) + assert.Equal(t, dsdk.Started, findFlow(t, id).State) } -func Test_Suspend_Success(t *testing.T) { +func Test_Suspend_WhenNotExists(t *testing.T) { + rr := post(t, "/v1/dataflows/"+uuid.NewString()+"/suspend", dsdk.DataFlowSuspendMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusNotFound, rr.Code) +} - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) - store := postgres.NewStore(database) - err = store.Create(ctx, flow) - assert.NoError(t, err) +func Test_Suspend_WhenNotStarted(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Prepared, true) + rr := post(t, "/v1/dataflows/"+id+"/suspend", dsdk.DataFlowSuspendMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusBadRequest, rr.Code) +} - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/suspend", strings.NewReader("")) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Resume_RequiresBody(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Suspended, false) + assert.Equal(t, http.StatusBadRequest, postRaw(t, "/v1/dataflows/"+id+"/resume", "").Code) + assert.Equal(t, http.StatusBadRequest, post(t, "/v1/dataflows/"+id+"/resume", map[string]any{}).Code) +} - assert.Equal(t, http.StatusOK, rr.Code) +func Test_Resume_WhenNotSuspended(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Started, false) + rr := post(t, "/v1/dataflows/"+id+"/resume", dsdk.DataFlowResumeMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusBadRequest, rr.Code) +} + +func Test_Resume_SuspendedLocally(t *testing.T) { + id := uuid.NewString() + flow, err := newFlowBuilder(id, pullProfile).State(dsdk.Suspended).SuspendedBy(dsdk.SuspendedByLocal).Build() + require.NoError(t, err) + require.NoError(t, postgres.NewStore(database).Create(ctx, flow)) - byId, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.Equal(t, dsdk.Suspended, byId.State) + rr := post(t, "/v1/dataflows/"+id+"/resume", dsdk.DataFlowResumeMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Equal(t, dsdk.Suspended, findFlow(t, id).State) } -func Test_Suspend_WithReason(t *testing.T) { +func Test_Resume_WhenNotFound(t *testing.T) { + rr := post(t, "/v1/dataflows/"+uuid.NewString()+"/resume", dsdk.DataFlowResumeMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusNotFound, rr.Code) +} - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) - store := postgres.NewStore(database) - err = store.Create(ctx, flow) - assert.NoError(t, err) +// --- terminate --- - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/suspend", strings.NewReader(`{"reason": "test reason"}`)) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Terminate_Success(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Started, false) + rr := post(t, "/v1/dataflows/"+id+"/terminate", dsdk.DataFlowTerminateMessage{MessageID: uuid.NewString(), Reason: "violation"}) assert.Equal(t, http.StatusOK, rr.Code) - - byId, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.Equal(t, dsdk.Suspended, byId.State) - assert.Equal(t, "test reason", byId.ErrorDetail) + flow := findFlow(t, id) + assert.Equal(t, dsdk.Terminated, flow.State) + assert.Equal(t, "violation", flow.ErrorDetail) } -func Test_Suspend_WhenNotExists(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) - //missing: storing the flow +func Test_Terminate_RequiresBody(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Started, false) + assert.Equal(t, http.StatusBadRequest, postRaw(t, "/v1/dataflows/"+id+"/terminate", "").Code) + assert.Equal(t, http.StatusBadRequest, post(t, "/v1/dataflows/"+id+"/terminate", map[string]any{"reason": "x"}).Code) + assert.Equal(t, dsdk.Started, findFlow(t, id).State) +} - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/suspend", strings.NewReader("")) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Terminate_WhenCompleted(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Completed, false) + rr := post(t, "/v1/dataflows/"+id+"/terminate", dsdk.DataFlowTerminateMessage{MessageID: uuid.NewString()}) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Equal(t, dsdk.Completed, findFlow(t, id).State) +} +func Test_Terminate_WhenNotFound(t *testing.T) { + rr := post(t, "/v1/dataflows/"+uuid.NewString()+"/terminate", dsdk.DataFlowTerminateMessage{MessageID: uuid.NewString()}) assert.Equal(t, http.StatusNotFound, rr.Code) } -func Test_Suspend_WhenNotStarted(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Completed).Build() // completed flows cannot transition to suspended - assert.NoError(t, err) - err = postgres.NewStore(database).Create(ctx, flow) - assert.NoError(t, err) +// --- completed --- - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/suspend", strings.NewReader("")) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_Complete(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Started, false) - assert.Equal(t, http.StatusBadRequest, rr.Code) + rr := postRaw(t, "/v1/dataflows/"+id+"/completed", "") + assert.Equal(t, http.StatusOK, rr.Code) + assert.Equal(t, dsdk.Completed, findFlow(t, id).State) } -func Test_Terminate_Success(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) - store := postgres.NewStore(database) - err = store.Create(ctx, flow) - assert.NoError(t, err) - - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/terminate", strings.NewReader("")) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusOK, rr.Code) - byId, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.Equal(t, dsdk.Terminated, byId.State) +func Test_Complete_NotFound(t *testing.T) { + rr := postRaw(t, "/v1/dataflows/"+uuid.NewString()+"/completed", "") + assert.Equal(t, http.StatusNotFound, rr.Code) +} +func Test_Complete_WrongState(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Prepared, true) + rr := postRaw(t, "/v1/dataflows/"+id+"/completed", "") + assert.Equal(t, http.StatusBadRequest, rr.Code) } -func Test_Terminate_WithReason(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) - store := postgres.NewStore(database) - err = store.Create(ctx, flow) - assert.NoError(t, err) +// --- status --- - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/terminate", strings.NewReader(`{"reason": "test reason"}`)) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +func Test_GetStatus(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Started, false) + rr := get(t, "/v1/dataflows/"+id+"/status") assert.Equal(t, http.StatusOK, rr.Code) - byId, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.Equal(t, dsdk.Terminated, byId.State) - assert.Equal(t, "test reason", byId.ErrorDetail) + var status dsdk.DataFlowStatusResponseMessage + require.NoError(t, json.NewDecoder(rr.Body).Decode(&status)) + assert.Equal(t, id, status.DataFlowID) + assert.Equal(t, dsdk.Started, status.State) } -func Test_Terminate_WhenNotFound(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) +func Test_GetStatus_WireFormat(t *testing.T) { + id := uuid.NewString() + createFlow(t, id, pullProfile, dsdk.Initialized, false) - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/terminate", strings.NewReader("")) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) + rr := get(t, "/v1/dataflows/"+id+"/status") + var raw map[string]any + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &raw)) + assert.Equal(t, map[string]any{"dataFlowId": id, "state": "INITIALIZED"}, raw, "only the DataFlowStatusResponseMessage fields") +} +func Test_GetStatus_NotFound(t *testing.T) { + rr := get(t, "/v1/dataflows/"+uuid.NewString()+"/status") assert.Equal(t, http.StatusNotFound, rr.Code) } -func Test_Complete(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) - store := postgres.NewStore(database) - err = store.Create(ctx, flow) - assert.NoError(t, err) - - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/completed", strings.NewReader("")) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +// --- routing --- - assert.Equal(t, http.StatusOK, rr.Code) - byId, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.Equal(t, dsdk.Completed, byId.State) +func Test_Router_MethodNotAllowed(t *testing.T) { + rr := get(t, "/v1/dataflows/prepare") + assert.Equal(t, http.StatusMethodNotAllowed, rr.Code) } -func Test_Complete_NotFound(t *testing.T) { - req, err := http.NewRequest(http.MethodPost, "/dataflows/not-exist/completed", strings.NewReader("")) - assert.NoError(t, err) +func Test_Router_UnknownPath(t *testing.T) { + rr := postRaw(t, "/dataflows/prepare", "{}") + assert.Equal(t, http.StatusNotFound, rr.Code) +} + +func Test_Router_CustomBasePath(t *testing.T) { + sdk, err := newSdk(database) + require.NoError(t, err) + custom := dsdk.NewHandler(sdk, dsdk.WithBasePath("/api/signaling/")) + + msg := newStartMessage(pullProfile, nil) + msg.Metadata = map[string]any{asyncKey: true} + payload, err := json.Marshal(msg) + require.NoError(t, err) + req, err := http.NewRequest(http.MethodPost, "/api/signaling/dataflows/start", bytes.NewBuffer(payload)) + require.NoError(t, err) rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) + custom.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusAccepted, rr.Code) + assert.Equal(t, "/api/signaling/dataflows/"+msg.DataFlowID+"/status", rr.Header().Get("Location")) + req, err = http.NewRequest(http.MethodGet, "/v1/dataflows/"+msg.DataFlowID+"/status", nil) + require.NoError(t, err) + rr = httptest.NewRecorder() + custom.ServeHTTP(rr, req) assert.Equal(t, http.StatusNotFound, rr.Code) } -func Test_Complete_WrongState(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Terminated).Build() - assert.NoError(t, err) - store := postgres.NewStore(database) - err = store.Create(ctx, flow) - assert.NoError(t, err) - - req, err := http.NewRequest(http.MethodPost, "/dataflows/"+flow.ID+"/completed", strings.NewReader("")) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) +// --- helpers --- - assert.Equal(t, http.StatusBadRequest, rr.Code) - byId, err := store.FindById(ctx, id) - assert.NoError(t, err) - assert.NotEqual(t, dsdk.Completed, byId.State) +func post(t *testing.T, path string, body any) *httptest.ResponseRecorder { + t.Helper() + payload, err := json.Marshal(body) + require.NoError(t, err) + return postRaw(t, path, string(payload)) } -func Test_GetStatus(t *testing.T) { - id := uuid.New().String() - flow, err := newFlowBuilder().ID(id).State(dsdk.Started).Build() - assert.NoError(t, err) - store := postgres.NewStore(database) - err = store.Create(ctx, flow) - assert.NoError(t, err) - - req, err := http.NewRequest(http.MethodGet, "/dataflows/"+flow.ID+"/status", nil) +func postRaw(t *testing.T, path string, body string) *httptest.ResponseRecorder { + t.Helper() + req, err := http.NewRequest(http.MethodPost, path, strings.NewReader(body)) + require.NoError(t, err) rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusOK, rr.Code) - var responseMessage dsdk.DataFlowStatusResponseMessage - err = json.NewDecoder(rr.Body).Decode(&responseMessage) - assert.NoError(t, err) - assert.Equal(t, responseMessage.State, dsdk.Started) + return rr } -func Test_GetStatus_NotFound(t *testing.T) { - - req, err := http.NewRequest(http.MethodGet, "/dataflows/not-exist/status", nil) - assert.NoError(t, err) +func get(t *testing.T, path string) *httptest.ResponseRecorder { + t.Helper() + req, err := http.NewRequest(http.MethodGet, path, nil) + require.NoError(t, err) rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) + return rr +} - assert.Equal(t, http.StatusNotFound, rr.Code) +func decodeStatus(t *testing.T, rr *httptest.ResponseRecorder) dsdk.DataFlowStatusMessage { + t.Helper() + assert.Equal(t, "application/json", rr.Header().Get("Content-Type")) + var status dsdk.DataFlowStatusMessage + require.NoError(t, json.NewDecoder(rr.Body).Decode(&status), rr.Body.String()) + return status } -func newFlowBuilder() *dsdk.DataFlowBuilder { - bldr := &dsdk.DataFlowBuilder{} - return bldr.ID("test-id"). - ParticipantID(uuid.New().String()). - DataspaceContext(uuid.New().String()). - CounterpartyID(uuid.New().String()). - CallbackAddress(newCallback()). - TransferType(newTransferType()) -} - -func serialize(obj any) ([]byte, error) { - marshal, err := json.Marshal(obj) - return marshal, err -} - -func newStartMessage() dsdk.DataFlowStartMessage { - return dsdk.DataFlowStartMessage{ - DataFlowBaseMessage: dsdk.DataFlowBaseMessage{ - MessageID: uuid.New().String(), - ParticipantID: uuid.New().String(), - CounterPartyID: uuid.New().String(), - DataspaceContext: uuid.New().String(), - ProcessID: uuid.New().String(), - AgreementID: uuid.New().String(), - DatasetID: uuid.New().String(), - CallbackAddress: newCallback(), - TransferType: newTransferType(), - DataAddress: &dsdk.DataAddress{ - Properties: map[string]any{ - "foo": "bar", - }, - }, - }, - } +func findFlow(t *testing.T, id string) *dsdk.DataFlow { + t.Helper() + flow, err := postgres.NewStore(database).FindById(ctx, id) + require.NoError(t, err) + return flow } -func newStartByIdMessage() dsdk.DataFlowStartedNotificationMessage { - return dsdk.DataFlowStartedNotificationMessage{ - DataAddress: &dsdk.DataAddress{ - Properties: map[string]any{ - "foo": "bar", - }, - }, +func assertNoFlow(t *testing.T, id string) { + t.Helper() + _, err := postgres.NewStore(database).FindById(ctx, id) + assert.ErrorIs(t, err, dsdk.ErrNotFound) +} + +func createFlow(t *testing.T, id string, profile string, state dsdk.DataFlowState, consumer bool) { + t.Helper() + flow, err := newFlowBuilder(id, profile).State(state).Consumer(consumer).Build() + require.NoError(t, err) + if state == dsdk.Suspended { + flow.SuspendedBy = dsdk.SuspendedByRemote + } + if !consumer && profile == pullProfile { + flow.SourceDataAddress = newDataAddress("https://provider.example.com/data") } + require.NoError(t, postgres.NewStore(database).Create(ctx, flow)) } -func newTransferType() dsdk.TransferType { - return dsdk.TransferType{ - DestinationType: "com.test.http", - FlowType: "pull", +func newFlowBuilder(id string, profile string) *dsdk.DataFlowBuilder { + return dsdk.NewDataFlowBuilder(). + ID(id). + ParticipantID("participant-id"). + DataspaceContext("dscontext"). + CounterpartyID("counterparty-id"). + AgreementID("agreement-id"). + DatasetID("dataset-id"). + Profile(profile). + Claims(map[string]any{"membership": "active"}) +} + +func newBaseMessage(profile string) dsdk.DataFlowBaseMessage { + return dsdk.DataFlowBaseMessage{ + MessageID: uuid.NewString(), + ParticipantID: uuid.NewString(), + CounterPartyID: uuid.NewString(), + DataspaceContext: uuid.NewString(), + DataFlowID: uuid.NewString(), + AgreementID: uuid.NewString(), + DatasetID: uuid.NewString(), + Profile: profile, + Claims: map[string]any{"membership": "active"}, } } -func newPrepareMessage() dsdk.DataFlowPrepareMessage { - return dsdk.DataFlowPrepareMessage{ - DataFlowBaseMessage: dsdk.DataFlowBaseMessage{ - MessageID: uuid.New().String(), - ParticipantID: uuid.New().String(), - CounterPartyID: uuid.New().String(), - DataspaceContext: uuid.New().String(), - ProcessID: uuid.New().String(), - AgreementID: uuid.New().String(), - DatasetID: uuid.New().String(), - CallbackAddress: newCallback(), - TransferType: newTransferType(), - DataAddress: &dsdk.DataAddress{}, - }, +func newPrepareMessage(profile string) dsdk.DataFlowPrepareMessage { + return dsdk.DataFlowPrepareMessage{DataFlowBaseMessage: newBaseMessage(profile)} +} + +func newStartMessage(profile string, address *dsdk.DataAddress) dsdk.DataFlowStartMessage { + return dsdk.DataFlowStartMessage{DataFlowBaseMessage: newBaseMessage(profile), DataAddress: address} +} + +func newDataAddress(endpoint string) *dsdk.DataAddress { + return &dsdk.DataAddress{ + EndpointType: "https://w3id.org/idsa/v4.1/HTTP", + Endpoint: endpoint, + EndpointProperties: []dsdk.EndpointProperty{{Name: "authType", Value: "bearer"}}, } } -func newCallback() dsdk.CallbackURL { - return dsdk.CallbackURL{Scheme: "http", Host: "test.com", Path: "/callback"} +func isAsync(flow *dsdk.DataFlow) bool { + async, _ := flow.Metadata[asyncKey].(bool) + return async } func newSdk(db *sql.DB) (*dsdk.DataPlaneSDK, error) { - sdk, err := dsdk.NewDataPlaneSDK( + return dsdk.NewDataPlaneSDK( dsdk.WithStore(postgres.NewStore(db)), dsdk.WithTransactionContext(postgres.NewDBTransactionContext(db)), + dsdk.WithPrepareProcessor(func(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { + if isAsync(flow) { + return &dsdk.DataFlowStatusMessage{State: dsdk.Preparing}, nil + } + if flow.IsPush() { + return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared, DataAddress: newDataAddress("https://consumer.example.com/push")}, nil + } + return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared}, nil + }), + dsdk.WithStartProcessor(func(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { + if isAsync(flow) { + return &dsdk.DataFlowStatusMessage{State: dsdk.Starting}, nil + } + if !flow.Consumer && flow.IsPull() { + address, err := dsdk.NewDataAddressBuilder(). + EndpointType("https://w3id.org/idsa/v4.1/HTTP"). + Endpoint("https://provider.example.com/data"). + Property("authorization", "token-"+flow.ID). + Build() + if err != nil { + return nil, err + } + return &dsdk.DataFlowStatusMessage{State: dsdk.Started, DataAddress: address}, nil + } + return &dsdk.DataFlowStatusMessage{State: dsdk.Started}, nil + }), ) - return sdk, err } diff --git a/pkg/dsdk/api.go b/pkg/dsdk/api.go index fb4e70e..c8e566d 100644 --- a/pkg/dsdk/api.go +++ b/pkg/dsdk/api.go @@ -13,11 +13,9 @@ package dsdk import ( - "bytes" "encoding/json" "errors" "fmt" - "io" "net/http" "github.com/google/uuid" @@ -26,248 +24,200 @@ import ( const contentType = "Content-Type" const jsonContentType = "application/json" +// DataPlaneApi implements the data plane endpoints of the Data Plane Signaling API. Its handlers expect the data flow +// id in the "id" path value; use NewHandler to mount them. type DataPlaneApi struct { - sdk *DataPlaneSDK + sdk *DataPlaneSDK + basePath string } func NewDataPlaneApi(sdk *DataPlaneSDK) *DataPlaneApi { - return &DataPlaneApi{sdk: sdk} + return &DataPlaneApi{sdk: sdk, basePath: DefaultBasePath} } +// Prepare handles POST /dataflows/prepare. func (d *DataPlaneApi) Prepare(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Invalid request method", http.StatusBadRequest) + var message DataFlowPrepareMessage + if !d.decode(w, r, &message) { return } - var prepareMessage DataFlowPrepareMessage - - if err := json.NewDecoder(r.Body).Decode(&prepareMessage); err != nil { - d.decodingError(w, err) - return - } - - if err := prepareMessage.Validate(); err != nil { + if err := message.Validate(); err != nil { d.handleError(err, w) return } - response, err := d.sdk.Prepare(r.Context(), prepareMessage) + response, err := d.sdk.Prepare(r.Context(), message) if err != nil { d.handleError(err, w) return } - var code int - if response.State == Prepared { - code = http.StatusOK - } else { + code := http.StatusOK + if response.State == Preparing { code = http.StatusAccepted + w.Header().Set("Location", d.statusPath(response.DataFlowID)) } d.writeResponse(w, code, response) } +// Start handles POST /dataflows/start. func (d *DataPlaneApi) Start(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Invalid request method", http.StatusBadRequest) + var message DataFlowStartMessage + if !d.decode(w, r, &message) { return } - var startMessage DataFlowStartMessage - - if err := json.NewDecoder(r.Body).Decode(&startMessage); err != nil { - d.decodingError(w, err) - return - } - - if err := startMessage.Validate(); err != nil { + if err := message.Validate(); err != nil { d.handleError(err, w) return } - response, err := d.sdk.Start(r.Context(), startMessage) + response, err := d.sdk.Start(r.Context(), message) if err != nil { d.handleError(err, w) return } - var code int - if response.State == Started { - code = http.StatusOK - } else { + code := http.StatusOK + if response.State == Starting { code = http.StatusAccepted - w.Header().Set("Location", "/dataflows/"+startMessage.ProcessID) + w.Header().Set("Location", d.statusPath(response.DataFlowID)) } d.writeResponse(w, code, response) - } -func (d *DataPlaneApi) StartById(w http.ResponseWriter, r *http.Request, id string) { - if r.Method != http.MethodPost { - http.Error(w, "Invalid request method", http.StatusBadRequest) +// Started handles POST /dataflows/{id}/started. It always answers 200 with a DataFlowStatusMessage on success. +func (d *DataPlaneApi) Started(w http.ResponseWriter, r *http.Request) { + var message DataFlowStartedNotificationMessage + if !d.decode(w, r, &message) { return } - var startMessage DataFlowStartedNotificationMessage - - if err := json.NewDecoder(r.Body).Decode(&startMessage); err != nil { - d.decodingError(w, err) - return - } - - if err := startMessage.Validate(); err != nil { + if err := message.Validate(); err != nil { d.handleError(err, w) return } - response, err := d.sdk.StartById(r.Context(), id, startMessage) + response, err := d.sdk.Started(r.Context(), r.PathValue("id"), message) if err != nil { d.handleError(err, w) return } - - var code int - if response.State == Started { - code = http.StatusOK - } else { - code = http.StatusAccepted - w.Header().Set("Location", "/dataflows/"+id) - } - d.writeResponse(w, code, response) + d.writeResponse(w, http.StatusOK, response) } -func (d *DataPlaneApi) Terminate(id string, w http.ResponseWriter, r *http.Request) { - reason := "" - // Peek into the body - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - d.decodingError(w, err) +// Suspend handles POST /dataflows/{id}/suspend. +func (d *DataPlaneApi) Suspend(w http.ResponseWriter, r *http.Request) { + var message DataFlowSuspendMessage + if !d.decode(w, r, &message) { return } - // if a body was sent, parse it, read the reason - - if len(bodyBytes) > 0 { - var terminateMessage DataFlowTransitionMessage - - if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&terminateMessage); err != nil { - d.decodingError(w, err) - return - } - if err := terminateMessage.Validate(); err != nil { - d.handleError(err, w) - return - } - reason = terminateMessage.Reason + if err := message.Validate(); err != nil { + d.handleError(err, w) + return } - terminateError := d.sdk.Terminate(r.Context(), id, reason) - if terminateError != nil { - d.handleError(terminateError, w) + if err := d.sdk.Suspend(r.Context(), r.PathValue("id"), message); err != nil { + d.handleError(err, w) return } - - w.Header().Set(contentType, jsonContentType) w.WriteHeader(http.StatusOK) } -func (d *DataPlaneApi) Suspend(id string, w http.ResponseWriter, r *http.Request) { - - reason := "" - // Peek into the body - bodyBytes, err := io.ReadAll(r.Body) +// Resume handles POST /dataflows/{id}/resume. +func (d *DataPlaneApi) Resume(w http.ResponseWriter, r *http.Request) { + var message DataFlowResumeMessage + if !d.decode(w, r, &message) { + return + } + if err := message.Validate(); err != nil { + d.handleError(err, w) + return + } + response, err := d.sdk.Resume(r.Context(), r.PathValue("id"), message) if err != nil { - d.decodingError(w, err) + d.handleError(err, w) return } - // if a body was sent, parse it, read the reason - if len(bodyBytes) > 0 { - var suspendMessage DataFlowTransitionMessage + d.writeResponse(w, http.StatusOK, response) +} - if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&suspendMessage); err != nil { - d.decodingError(w, err) - return - } - if err := suspendMessage.Validate(); err != nil { - d.handleError(err, w) - return - } - reason = suspendMessage.Reason +// Terminate handles POST /dataflows/{id}/terminate. +func (d *DataPlaneApi) Terminate(w http.ResponseWriter, r *http.Request) { + var message DataFlowTerminateMessage + if !d.decode(w, r, &message) { + return } - - suspensionError := d.sdk.Suspend(r.Context(), id, reason) - if suspensionError != nil { - d.handleError(suspensionError, w) + if err := message.Validate(); err != nil { + d.handleError(err, w) + return + } + if err := d.sdk.Terminate(r.Context(), r.PathValue("id"), message); err != nil { + d.handleError(err, w) return } - - w.Header().Set(contentType, jsonContentType) w.WriteHeader(http.StatusOK) - } -func (d *DataPlaneApi) Status(processID string, w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Invalid request method", http.StatusBadRequest) +// Completed handles POST /dataflows/{id}/completed. The request has an empty body. +func (d *DataPlaneApi) Completed(w http.ResponseWriter, r *http.Request) { + if err := d.sdk.Complete(r.Context(), r.PathValue("id")); err != nil { + d.handleError(err, w) return } - dataFlow, err := d.sdk.Status(r.Context(), processID) + w.WriteHeader(http.StatusOK) +} + +// Status handles GET /dataflows/{id}/status. +func (d *DataPlaneApi) Status(w http.ResponseWriter, r *http.Request) { + dataFlow, err := d.sdk.Status(r.Context(), r.PathValue("id")) if err != nil { d.handleError(err, w) return } - w.Header().Set(contentType, jsonContentType) response := DataFlowStatusResponseMessage{ - State: dataFlow.State, DataFlowID: dataFlow.ID, + State: dataFlow.State, } d.writeResponse(w, http.StatusOK, response) } -func (d *DataPlaneApi) Complete(processID string, w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Invalid request method", http.StatusBadRequest) - return - } - err := d.sdk.Complete(r.Context(), processID) - if err != nil { - d.handleError(err, w) - return - } - d.writeResponse(w, http.StatusOK, nil) +func (d *DataPlaneApi) statusPath(id string) string { + return d.basePath + "/dataflows/" + id + "/status" } -func (d *DataPlaneApi) decodingError(w http.ResponseWriter, err error) { - id := uuid.NewString() - d.sdk.Monitor.Printf("Error decoding flow [%s]: %v\n", id, err) - d.writeResponse(w, http.StatusBadRequest, &DataFlowResponseMessage{Error: fmt.Sprintf("Failed to decode request body [%s]", id)}) +func (d *DataPlaneApi) decode(w http.ResponseWriter, r *http.Request, target any) bool { + if err := json.NewDecoder(r.Body).Decode(target); err != nil { + id := uuid.NewString() + d.sdk.Monitor.Printf("Error decoding request [%s]: %v\n", id, err) + d.writeResponse(w, http.StatusBadRequest, &ErrorResponse{Error: fmt.Sprintf("Failed to decode request body [%s]", id)}) + return false + } + return true } -// handleError writes an error message to the HTTP response that indicates "any other" error, such as 409, 500, etc. +// handleError maps an SDK error to an HTTP status code and writes an ErrorResponse. func (d *DataPlaneApi) handleError(err error, w http.ResponseWriter) { - switch { case errors.Is(err, ErrValidation), errors.Is(err, ErrInvalidTransition), errors.Is(err, ErrInvalidInput): - d.badRequest(err.Error(), w) + d.writeResponse(w, http.StatusBadRequest, &ErrorResponse{Error: err.Error()}) case errors.Is(err, ErrNotFound): - d.writeResponse(w, http.StatusNotFound, &DataFlowResponseMessage{Error: err.Error()}) + d.writeResponse(w, http.StatusNotFound, &ErrorResponse{Error: err.Error()}) case errors.Is(err, ErrConflict): - message := fmt.Sprintf("%s", err) - d.writeResponse(w, http.StatusConflict, &DataFlowResponseMessage{Error: message}) + d.writeResponse(w, http.StatusConflict, &ErrorResponse{Error: err.Error()}) default: message := fmt.Sprintf("Error processing flow: %s", err) d.sdk.Monitor.Println(message) - d.writeResponse(w, http.StatusInternalServerError, &DataFlowResponseMessage{Error: message}) + d.writeResponse(w, http.StatusInternalServerError, &ErrorResponse{Error: message}) } } -func (d *DataPlaneApi) badRequest(errMsg string, w http.ResponseWriter) { - d.writeResponse(w, http.StatusBadRequest, &DataFlowResponseMessage{Error: errMsg}) -} - func (d *DataPlaneApi) writeResponse(w http.ResponseWriter, code int, response any) { - w.Header().Set(contentType, jsonContentType) - w.WriteHeader(code) - if err := json.NewEncoder(w).Encode(response); err != nil { + body, err := json.Marshal(response) + if err != nil { id := uuid.NewString() - message := fmt.Sprintf("Error encoding response [%s]", id) - d.sdk.Monitor.Println(message) - d.writeResponse(w, http.StatusInternalServerError, &DataFlowResponseMessage{Error: message}) - return + d.sdk.Monitor.Printf("Error encoding response [%s]: %v\n", id, err) + code = http.StatusInternalServerError + body, _ = json.Marshal(&ErrorResponse{Error: fmt.Sprintf("Error encoding response [%s]", id)}) } + w.Header().Set(contentType, jsonContentType) + w.WriteHeader(code) + _, _ = w.Write(append(body, '\n')) } diff --git a/pkg/dsdk/dsdk.go b/pkg/dsdk/dsdk.go index c84e5fe..7fa1a25 100644 --- a/pkg/dsdk/dsdk.go +++ b/pkg/dsdk/dsdk.go @@ -5,14 +5,20 @@ import ( "errors" "fmt" "log" + + "github.com/google/uuid" ) -// DataFlowProcessor is an extension point for handling SDK data flow events. Implementations may modify the data flow instance -// which will be persisted by the SDK. If the message is a duplicate, implementations must support idempotent behavior. -type DataFlowProcessor func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowResponseMessage, error) +// DataFlowProcessor is an extension point for handling SDK data flow events. Implementations may modify the data flow +// instance, which is persisted by the SDK, and return the resulting state, an optional DataAddress and an optional +// error description. The SDK sets the messageId and dataFlowId of the returned message. If the message is a duplicate, +// implementations must support idempotent behavior. +type DataFlowProcessor func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) type ProcessorOptions struct { - Duplicate bool + // Duplicate is true if the flow already reached the requested state and the message is processed again. + Duplicate bool + // DataAddress is the DataAddress received with the message, if any. DataAddress *DataAddress } @@ -23,6 +29,8 @@ type LogMonitor interface { Printf(format string, v ...any) } +var errNilResponse = errors.New("processor returned no response") + type DataPlaneSDK struct { Store DataplaneStore TrxContext TransactionContext @@ -30,226 +38,276 @@ type DataPlaneSDK struct { onPrepare DataFlowProcessor onStart DataFlowProcessor + onResume DataFlowProcessor onTerminate DataFlowHandler onSuspend DataFlowHandler onComplete DataFlowHandler } // Prepare is called on the consumer to prepare for receiving data. -// It invokes the onPrepare callback and persists the created flow. Returns a response or an error if the process fails. -func (dsdk *DataPlaneSDK) Prepare(ctx context.Context, message DataFlowPrepareMessage) (*DataFlowResponseMessage, error) { - processID := message.ProcessID - if processID == "" { - return nil, errors.New("processID cannot be empty") - } - var response *DataFlowResponseMessage - err := dsdk.execute(ctx, func(context.Context) error { - flow, err := dsdk.Store.FindById(ctx, processID) +// It invokes the onPrepare processor and persists the created flow. For push transfers the response carries the +// DataAddress the provider pushes data to. +func (dsdk *DataPlaneSDK) Prepare(ctx context.Context, message DataFlowPrepareMessage) (*DataFlowStatusMessage, error) { + if err := message.Validate(); err != nil { + return nil, err + } + id := message.DataFlowID + var response *DataFlowStatusMessage + err := dsdk.execute(ctx, func(ctx context.Context) error { + flow, err := dsdk.Store.FindById(ctx, id) if err != nil && !errors.Is(err, ErrNotFound) { - return fmt.Errorf("performing de-duplication for %s: %w", processID, err) + return fmt.Errorf("performing de-duplication for %s: %w", id, err) } switch { case flow != nil && (flow.State == Preparing || flow.State == Prepared): - // duplicate message, pass to handler to generate a data address if needed (on consumer) - response, err = dsdk.onPrepare(ctx, flow, dsdk, &ProcessorOptions{Duplicate: true}) + // duplicate message, pass to processor to generate a data address if needed + response, err = dsdk.process(ctx, dsdk.onPrepare, flow, &ProcessorOptions{Duplicate: true}) if err != nil { - return fmt.Errorf("processing data flow: %w", err) + return err + } + if err := dsdk.prepareState(response, flow); err != nil { + return err } - // todo: not sure about this, added because Prepare() has it too if err := dsdk.Store.Save(ctx, flow); err != nil { - return fmt.Errorf("creating data flow: %w", err) + return fmt.Errorf("saving data flow %s: %w", flow.ID, err) } return nil case flow != nil: - return fmt.Errorf("%w: data flow %s is not in PREPARING or PREPARED state but in %s", ErrConflict, flow.ID, flow.State.String()) - //return NewConflictError(fmt.Sprintf("data flow %s is not in PREPARING or PREPARED state", flow.ID)) - } - flow, err = NewDataFlowBuilder().ID(processID). - Consumer(true). - State(Preparing). - AgreementID(message.AgreementID). - DatasetID(message.DatasetID). - ParticipantID(message.ParticipantID). - CounterpartyID(message.CounterPartyID). - DataspaceContext(message.DataspaceContext). - TransferType(message.TransferType). - CallbackAddress(message.CallbackAddress). - Build() + return fmt.Errorf("%w: data flow %s is not in PREPARING or PREPARED state but in %s", ErrConflict, flow.ID, flow.State) + } + flow, err = newFlow(message.DataFlowBaseMessage, true) if err != nil { - return fmt.Errorf("creating data flow: %w", err) + return err } - response, err = dsdk.onPrepare(ctx, flow, dsdk, &ProcessorOptions{}) + response, err = dsdk.process(ctx, dsdk.onPrepare, flow, &ProcessorOptions{}) if err != nil { - return fmt.Errorf("processing data flow %s: %w", flow.ID, err) + return err } - if response.State == Prepared { - err := flow.TransitionToPrepared() - if err != nil { - return err - } - } else if response.State == Preparing { - err := flow.TransitionToPreparing() - if err != nil { - return err - } - } else { - return fmt.Errorf("onPrepare returned an invalid state %s", response.State) + if err := dsdk.prepareState(response, flow); err != nil { + return err } if err := dsdk.Store.Create(ctx, flow); err != nil { return fmt.Errorf("creating data flow %s: %w", flow.ID, err) } return nil }) - - // fixme: shouldn't we always return a clean nil/error or response/nil tuple? - return response, err + if err != nil { + return nil, err + } + return response, nil } -// Start is called on the provider and starts a data flow based on the given start message and execution context. -// It invokes the onStart callback and persists the created flow. Returns a response or an error if the process fails. -func (dsdk *DataPlaneSDK) Start(ctx context.Context, message DataFlowStartMessage) (*DataFlowResponseMessage, error) { - processID := message.ProcessID - if processID == "" { - return nil, errors.New("processID cannot be empty") +// Start is called on the provider and starts a data flow based on the given start message. +// It invokes the onStart processor and persists the created flow. For pull transfers the response carries the +// DataAddress the consumer obtains the data from. A start message for an existing data flow is rejected with ErrConflict. +func (dsdk *DataPlaneSDK) Start(ctx context.Context, message DataFlowStartMessage) (*DataFlowStatusMessage, error) { + if err := message.Validate(); err != nil { + return nil, err } - var response *DataFlowResponseMessage - err := dsdk.execute(ctx, func(context.Context) error { - flow, err := dsdk.Store.FindById(ctx, processID) + id := message.DataFlowID + var response *DataFlowStatusMessage + err := dsdk.execute(ctx, func(ctx context.Context) error { + existing, err := dsdk.Store.FindById(ctx, id) if err != nil && !errors.Is(err, ErrNotFound) { - return fmt.Errorf("performing de-duplication for %s: %w", processID, err) - } - - if flow == nil { - // provider side, process - flow, err = NewDataFlowBuilder().ID(processID). - State(Starting). - AgreementID(message.AgreementID). - DatasetID(message.DatasetID). - ParticipantID(message.ParticipantID). - CounterpartyID(message.CounterPartyID). - DataspaceContext(message.DataspaceContext). - TransferType(message.TransferType). - CallbackAddress(message.CallbackAddress). - Build() - if err != nil { - return fmt.Errorf("creating data flow: %w", err) - } - response, err = dsdk.onStart(ctx, flow, dsdk, &ProcessorOptions{DataAddress: message.DataAddress}) - if err != nil { - return fmt.Errorf("processing data flow: %w", err) - } - - err = dsdk.startState(response, flow) - if err != nil { - return fmt.Errorf("onStart returned an invalid state: %w", err) - } + return fmt.Errorf("performing de-duplication for %s: %w", id, err) + } + if existing != nil { + return fmt.Errorf("%w: data flow %s already exists", ErrConflict, id) + } - if err := dsdk.Store.Create(ctx, flow); err != nil { - return fmt.Errorf("creating data flow: %w", err) - } - return nil + flow, err := newFlow(message.DataFlowBaseMessage, false) + if err != nil { + return err + } + switch { + case flow.IsPull() && message.DataAddress != nil: + return fmt.Errorf("%w: a start message for a pull transfer must not contain a data address", ErrInvalidInput) + case flow.IsPush() && message.DataAddress == nil: + return fmt.Errorf("%w: a start message for a push transfer must contain a data address", ErrInvalidInput) } + dsdk.setReceivedAddress(flow, message.DataAddress) - response, err = dsdk.startExistingFlow(ctx, flow, message.DataAddress) - return err + response, err = dsdk.process(ctx, dsdk.onStart, flow, &ProcessorOptions{DataAddress: message.DataAddress}) + if err != nil { + return err + } + if err := dsdk.startState(response, flow); err != nil { + return err + } + if err := dsdk.Store.Create(ctx, flow); err != nil { + return fmt.Errorf("creating data flow %s: %w", flow.ID, err) + } + return nil }) - - return response, err - + if err != nil { + return nil, err + } + return response, nil } -func (dsdk *DataPlaneSDK) StartById(ctx context.Context, processID string, message DataFlowStartedNotificationMessage) (*DataFlowResponseMessage, error) { - var response *DataFlowResponseMessage - +// Started is called on the consumer when the provider has started the transfer. For pull transfers the message must +// carry the DataAddress to obtain the data from; for push transfers it must not carry one. +// It invokes the onStart processor and persists the flow. +func (dsdk *DataPlaneSDK) Started(ctx context.Context, id string, message DataFlowStartedNotificationMessage) (*DataFlowStatusMessage, error) { + if err := message.Validate(); err != nil { + return nil, err + } + var response *DataFlowStatusMessage err := dsdk.execute(ctx, func(ctx context.Context) error { - existingFlow, err := dsdk.Store.FindById(ctx, processID) - if err != nil && !errors.Is(err, ErrNotFound) { - return fmt.Errorf("performing de-duplication for %s: %w", processID, err) + flow, err := dsdk.Store.FindById(ctx, id) + if err != nil { + return fmt.Errorf("starting data flow %s: %w", id, err) } - - if existingFlow == nil { // this should never happen -> the store would return an error - return ErrNotFound + if !flow.Consumer { + return fmt.Errorf("%w: started is only valid for consumer data flows", ErrInvalidInput) } - - if !existingFlow.Consumer { - return fmt.Errorf("%w: startById is only valid for consumer data flows", ErrInvalidInput) + switch { + case flow.IsPull() && message.DataAddress == nil: + return fmt.Errorf("%w: a started message for a pull transfer must contain a data address", ErrInvalidInput) + case flow.IsPush() && message.DataAddress != nil: + return fmt.Errorf("%w: a started message for a push transfer must not contain a data address", ErrInvalidInput) } - response, err = dsdk.startExistingFlow(ctx, existingFlow, message.DataAddress) - return err + var duplicate bool + switch flow.State { + case Started: + duplicate = true + case Prepared, Starting: + default: + return fmt.Errorf("%w: data flow %s cannot be started from state %s", ErrInvalidTransition, flow.ID, flow.State) + } + dsdk.setReceivedAddress(flow, message.DataAddress) + response, err = dsdk.process(ctx, dsdk.onStart, flow, &ProcessorOptions{Duplicate: duplicate, DataAddress: message.DataAddress}) + if err != nil { + return err + } + if err := dsdk.startState(response, flow); err != nil { + return err + } + // the consumer already handed out its DataAddress (if any) in the prepare response + response.DataAddress = nil + if err := dsdk.Store.Save(ctx, flow); err != nil { + return fmt.Errorf("saving data flow %s: %w", flow.ID, err) + } + return nil }) - return response, err - + if err != nil { + return nil, err + } + return response, nil } -func (dsdk *DataPlaneSDK) Terminate(ctx context.Context, processID string, reason string) error { - if processID == "" { - return errors.New("processID cannot be empty") +// Suspend suspends a STARTED data flow on request of the control plane. +func (dsdk *DataPlaneSDK) Suspend(ctx context.Context, id string, message DataFlowSuspendMessage) error { + if err := message.Validate(); err != nil { + return err } - return dsdk.execute(ctx, func(ctx context.Context) error { - flow, err := dsdk.Store.FindById(ctx, processID) + flow, err := dsdk.Store.FindById(ctx, id) if err != nil { - return fmt.Errorf("terminating data flow %s: %w", processID, err) + return fmt.Errorf("suspending data flow %s: %w", id, err) } - if Terminated == flow.State { + if flow.State == Suspended { return nil // duplicate message, skip processing } + if flow.State != Started { + return fmt.Errorf("%w: cannot transition from %v to SUSPENDED", ErrInvalidTransition, flow.State) + } - if err := dsdk.onTerminate(ctx, flow); err != nil { - return fmt.Errorf("terminating data flow %s: %w", flow.ID, err) + if err := dsdk.onSuspend(ctx, flow); err != nil { + return fmt.Errorf("suspending data flow %s: %w", flow.ID, err) + } + if err := flow.TransitionToSuspended(message.Reason, SuspendedByRemote); err != nil { + return err } + if err := dsdk.Store.Save(ctx, flow); err != nil { + return fmt.Errorf("suspending data flow %s: %w", flow.ID, err) + } + return nil + }) +} - err = flow.TransitionToTerminated(reason) +// Resume resumes a data flow that was suspended by the control plane. The message may carry a new DataAddress of the +// counterparty. The response carries the DataAddress this data plane provides to the counterparty, if any. +func (dsdk *DataPlaneSDK) Resume(ctx context.Context, id string, message DataFlowResumeMessage) (*DataFlowStatusMessage, error) { + if err := message.Validate(); err != nil { + return nil, err + } + var response *DataFlowStatusMessage + err := dsdk.execute(ctx, func(ctx context.Context) error { + flow, err := dsdk.Store.FindById(ctx, id) if err != nil { - return err + return fmt.Errorf("resuming data flow %s: %w", id, err) } - err = dsdk.Store.Save(ctx, flow) + if flow.State != Suspended { + return fmt.Errorf("%w: data flow %s cannot be resumed from state %s", ErrInvalidTransition, flow.ID, flow.State) + } + if flow.SuspendedBy != SuspendedByRemote { + return fmt.Errorf("%w: data flow %s was suspended by the %s party and can only be resumed by it", ErrInvalidTransition, flow.ID, flow.SuspendedBy) + } + + if message.DataAddress != nil { + dsdk.setReceivedAddress(flow, message.DataAddress) + } + response, err = dsdk.process(ctx, dsdk.onResume, flow, &ProcessorOptions{DataAddress: message.DataAddress}) if err != nil { - return fmt.Errorf("terminating data flow %s: %w", flow.ID, err) + return err + } + if response.State != Started { + return fmt.Errorf("onResume returned an invalid state %s", response.State) + } + if err := flow.TransitionToResumed(SuspendedByRemote); err != nil { + return err + } + if err := dsdk.Store.Save(ctx, flow); err != nil { + return fmt.Errorf("resuming data flow %s: %w", flow.ID, err) } return nil }) + if err != nil { + return nil, err + } + return response, nil } -func (dsdk *DataPlaneSDK) Suspend(ctx context.Context, processID string, reason string) error { - if processID == "" { - return errors.New("processID cannot be empty") +// Terminate terminates a data flow. COMPLETED flows cannot be terminated. +func (dsdk *DataPlaneSDK) Terminate(ctx context.Context, id string, message DataFlowTerminateMessage) error { + if err := message.Validate(); err != nil { + return err } - return dsdk.execute(ctx, func(ctx context.Context) error { - flow, err := dsdk.Store.FindById(ctx, processID) + flow, err := dsdk.Store.FindById(ctx, id) if err != nil { - return fmt.Errorf("suspending data flow %s: %w", processID, err) + return fmt.Errorf("terminating data flow %s: %w", id, err) } - if Suspended == flow.State { + if flow.State == Terminated { return nil // duplicate message, skip processing } + if flow.State == Completed { + return fmt.Errorf("%w: data flow %s is already COMPLETED", ErrInvalidTransition, flow.ID) + } - if err := dsdk.onSuspend(ctx, flow); err != nil { - return fmt.Errorf("suspending data flow %s: %w", flow.ID, err) + if err := dsdk.onTerminate(ctx, flow); err != nil { + return fmt.Errorf("terminating data flow %s: %w", flow.ID, err) } - err = flow.TransitionToSuspended(reason) - if err != nil { + if err := flow.TransitionToTerminated(message.Reason); err != nil { return err } - - err = dsdk.Store.Save(ctx, flow) - if err != nil { - return fmt.Errorf("suspending data flow %s: %w", flow.ID, err) + if err := dsdk.Store.Save(ctx, flow); err != nil { + return fmt.Errorf("terminating data flow %s: %w", flow.ID, err) } return nil }) - } +// Status returns the data flow with the given id. func (dsdk *DataPlaneSDK) Status(ctx context.Context, id string) (*DataFlow, error) { var flow *DataFlow err := dsdk.execute(ctx, func(ctx context.Context) error { @@ -263,94 +321,135 @@ func (dsdk *DataPlaneSDK) Status(ctx context.Context, id string) (*DataFlow, err return flow, err } -func (dsdk *DataPlaneSDK) Complete(ctx context.Context, dataflowID string) error { - if dataflowID == "" { - return errors.New("processID cannot be empty") +// Complete marks a STARTED data flow as COMPLETED and invokes the onComplete processor. +func (dsdk *DataPlaneSDK) Complete(ctx context.Context, id string) error { + if id == "" { + return fmt.Errorf("%w: data flow id cannot be empty", ErrInvalidInput) } return dsdk.execute(ctx, func(ctx context.Context) error { - flow, err := dsdk.Store.FindById(ctx, dataflowID) + flow, err := dsdk.Store.FindById(ctx, id) if err != nil { - return fmt.Errorf("completing data flow %s: %w", dataflowID, err) + return fmt.Errorf("completing data flow %s: %w", id, err) } if flow.State == Completed { // de-duplication return nil } - transitionError := flow.TransitionToCompleted() - if transitionError != nil { - return transitionError + if err := flow.TransitionToCompleted(); err != nil { + return err } // only invoked if the transition was successful - e := dsdk.onComplete(ctx, flow) - if e != nil { - return e + if err := dsdk.onComplete(ctx, flow); err != nil { + return fmt.Errorf("completing data flow %s: %w", flow.ID, err) } - storeErr := dsdk.Store.Save(ctx, flow) - if storeErr != nil { - return fmt.Errorf("completing data flow %s: %w", flow.ID, storeErr) + if err := dsdk.Store.Save(ctx, flow); err != nil { + return fmt.Errorf("completing data flow %s: %w", flow.ID, err) } return nil }) } -func (dsdk *DataPlaneSDK) startExistingFlow(ctx context.Context, flow *DataFlow, sourceAddress *DataAddress) (*DataFlowResponseMessage, error) { - switch { - case flow != nil && (flow.State == Starting || flow.State == Started): - // duplicate message, pass to handler to generate a data address if needed - response, err := dsdk.onStart(ctx, flow, dsdk, &ProcessorOptions{Duplicate: true, DataAddress: sourceAddress}) - if err != nil { - return nil, fmt.Errorf("processing data flow: %w", err) - } +func newFlow(message DataFlowBaseMessage, consumer bool) (*DataFlow, error) { + flow, err := NewDataFlowBuilder().ID(message.DataFlowID). + Consumer(consumer). + State(Initialized). + AgreementID(message.AgreementID). + DatasetID(message.DatasetID). + ParticipantID(message.ParticipantID). + CounterpartyID(message.CounterPartyID). + DataspaceContext(message.DataspaceContext). + Profile(message.Profile). + Claims(message.Claims). + Labels(message.Labels). + Metadata(message.Metadata). + Build() + if err != nil { + return nil, fmt.Errorf("creating data flow: %w", err) + } + return flow, nil +} - err = dsdk.startState(response, flow) - if err != nil { - return nil, fmt.Errorf("onStart returned an invalid state: %w", err) +// process invokes the processor, fills in the message and data flow ids, and keeps the DataAddress only where the +// specification allows this data plane to send one: consumer-push and provider-pull. The returned address is +// recorded on the flow. +func (dsdk *DataPlaneSDK) process(ctx context.Context, processor DataFlowProcessor, flow *DataFlow, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + response, err := processor(ctx, flow, dsdk, options) + if err != nil { + return nil, fmt.Errorf("processing data flow %s: %w", flow.ID, err) + } + if response == nil { + return nil, fmt.Errorf("processing data flow %s: %w", flow.ID, errNilResponse) + } + response.MessageID = uuid.NewString() + response.DataFlowID = flow.ID + if providesAddress(flow) { + if response.DataAddress != nil { + setProvidedAddress(flow, response.DataAddress) + } else { + response.DataAddress = providedAddress(flow).Copy() } + } else { + response.DataAddress = nil + } + return response, nil +} - if err := dsdk.Store.Save(ctx, flow); err != nil { - return nil, fmt.Errorf("creating data flow: %w", err) - } - return response, err - case flow != nil && flow.Consumer && flow.State == Prepared: - // consumer side, process - response, err := dsdk.onStart(ctx, flow, dsdk, &ProcessorOptions{DataAddress: sourceAddress}) - if err != nil { - return nil, fmt.Errorf("processing data flow: %w", err) - } +// providesAddress is true for flows where this data plane hands a DataAddress to the counterparty: the consumer +// receives pushed data, and the provider serves pulled data. +func providesAddress(flow *DataFlow) bool { + return (flow.Consumer && flow.IsPush()) || (!flow.Consumer && flow.IsPull()) +} - err = dsdk.startState(response, flow) - if err != nil { - return nil, fmt.Errorf("onStart returned an invalid state: %w", err) - } +func providedAddress(flow *DataFlow) *DataAddress { + if flow.Consumer { + return flow.DestinationDataAddress + } + return flow.SourceDataAddress +} - if err := dsdk.Store.Save(ctx, flow); err != nil { - return nil, fmt.Errorf("updating data flow: %w", err) - } +func setProvidedAddress(flow *DataFlow, address *DataAddress) { + if flow.Consumer { + flow.DestinationDataAddress = address.Copy() + } else { + flow.SourceDataAddress = address.Copy() + } +} - return response, nil +// setReceivedAddress records a DataAddress received from the counterparty: where the provider pushes to, or where the +// consumer pulls from. +func (dsdk *DataPlaneSDK) setReceivedAddress(flow *DataFlow, address *DataAddress) { + if address == nil { + return + } + if flow.Consumer { + flow.SourceDataAddress = address.Copy() + } else { + flow.DestinationDataAddress = address.Copy() + } +} +func (dsdk *DataPlaneSDK) prepareState(response *DataFlowStatusMessage, flow *DataFlow) error { + switch response.State { + case Prepared: + return flow.TransitionToPrepared() + case Preparing: + return flow.TransitionToPreparing() default: - return nil, fmt.Errorf("%w: data flow %s is not in STARTED state: %s", ErrInvalidTransition, flow.ID, flow.State) + return fmt.Errorf("onPrepare returned an invalid state %s", response.State) } } -func (dsdk *DataPlaneSDK) startState(response *DataFlowResponseMessage, flow *DataFlow) error { - if response.State == Started { - err := flow.TransitionToStarted() - if err != nil { - return err - } - } else if response.State == Starting { - err := flow.TransitionToStarting() - if err != nil { - return err - } - } else { +func (dsdk *DataPlaneSDK) startState(response *DataFlowStatusMessage, flow *DataFlow) error { + switch response.State { + case Started: + return flow.TransitionToStarted() + case Starting: + return flow.TransitionToStarting() + default: return fmt.Errorf("onStart returned an invalid state %s", response.State) } - return nil } func (dsdk *DataPlaneSDK) execute(ctx context.Context, callback func(ctx2 context.Context) error) error { @@ -389,12 +488,19 @@ func WithPrepareProcessor(processor DataFlowProcessor) DataPlaneSDKOption { } } +// WithStartProcessor sets the processor invoked for /start on the provider and for /started on the consumer. func WithStartProcessor(processor DataFlowProcessor) DataPlaneSDKOption { return func(sdk *DataPlaneSDK) { sdk.onStart = processor } } +func WithResumeProcessor(processor DataFlowProcessor) DataPlaneSDKOption { + return func(sdk *DataPlaneSDK) { + sdk.onResume = processor + } +} + func WithTerminateProcessor(handler DataFlowHandler) DataPlaneSDKOption { return func(sdk *DataPlaneSDK) { sdk.onTerminate = handler @@ -407,6 +513,12 @@ func WithSuspendProcessor(handler DataFlowHandler) DataPlaneSDKOption { } } +func WithCompleteProcessor(handler DataFlowHandler) DataPlaneSDKOption { + return func(sdk *DataPlaneSDK) { + sdk.onComplete = handler + } +} + func NewDataPlaneSDK(options ...DataPlaneSDKOption) (*DataPlaneSDK, error) { sdk := &DataPlaneSDK{} @@ -428,41 +540,37 @@ func NewDataPlaneSDK(options ...DataPlaneSDKOption) (*DataPlaneSDK, error) { sdk.Monitor = defaultLogMonitor{} } if sdk.onPrepare == nil { - sdk.onPrepare = func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{ - DataplaneID: "TODO_REPLACE_ME", - DataAddress: &flow.DestinationDataAddress, - State: Prepared, - Error: ""}, nil - } + sdk.onPrepare = stateProcessor(Prepared) } if sdk.onStart == nil { - sdk.onStart = func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{ - State: Started, - DataplaneID: "TODO_REPLACE_ME", - DataAddress: &flow.DestinationDataAddress, - Error: ""}, nil - } + sdk.onStart = stateProcessor(Started) + } + if sdk.onResume == nil { + sdk.onResume = stateProcessor(Started) } if sdk.onTerminate == nil { - sdk.onTerminate = func(context context.Context, flow *DataFlow) error { - return nil - } + sdk.onTerminate = noopHandler } if sdk.onSuspend == nil { - sdk.onSuspend = func(context context.Context, flow *DataFlow) error { - return nil - } + sdk.onSuspend = noopHandler } if sdk.onComplete == nil { - sdk.onComplete = func(context context.Context, flow *DataFlow) error { - return nil - } + sdk.onComplete = noopHandler } return sdk, nil } +// stateProcessor returns a processor that only reports the given state. +func stateProcessor(state DataFlowState) DataFlowProcessor { + return func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowStatusMessage, error) { + return &DataFlowStatusMessage{State: state}, nil + } +} + +func noopHandler(context.Context, *DataFlow) error { + return nil +} + type defaultLogMonitor struct { } diff --git a/pkg/dsdk/dsdk_options_test.go b/pkg/dsdk/dsdk_options_test.go index 1b3d17b..8b59b61 100644 --- a/pkg/dsdk/dsdk_options_test.go +++ b/pkg/dsdk/dsdk_options_test.go @@ -51,8 +51,8 @@ func Test_WithMonitor(t *testing.T) { } func Test_WithPrepareProcessor(t *testing.T) { - processor := func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: Prepared}, nil + processor := func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + return &DataFlowStatusMessage{State: Prepared}, nil } sdk := &DataPlaneSDK{} @@ -63,8 +63,8 @@ func Test_WithPrepareProcessor(t *testing.T) { } func Test_WithStartProcessor(t *testing.T) { - processor := func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: Started}, nil + processor := func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + return &DataFlowStatusMessage{State: Started}, nil } sdk := &DataPlaneSDK{} @@ -98,6 +98,30 @@ func Test_WithSuspendProcessor(t *testing.T) { require.NotNil(t, sdk.onSuspend) } +func Test_WithResumeProcessor(t *testing.T) { + processor := func(context context.Context, flow *DataFlow, sdk *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + return &DataFlowStatusMessage{State: Started}, nil + } + sdk := &DataPlaneSDK{} + + option := WithResumeProcessor(processor) + option(sdk) + + require.NotNil(t, sdk.onResume) +} + +func Test_WithCompleteProcessor(t *testing.T) { + handler := func(context.Context, *DataFlow) error { + return nil + } + sdk := &DataPlaneSDK{} + + option := WithCompleteProcessor(handler) + option(sdk) + + require.NotNil(t, sdk.onComplete) +} + func Test_NewDataPlaneSDK_WithoutOptionalFields(t *testing.T) { store := NewMockDataplaneStore(t) trxContext := &mockTrxContext{} @@ -116,6 +140,8 @@ func Test_NewDataPlaneSDK_WithoutOptionalFields(t *testing.T) { assert.NotNil(t, sdk.onStart) assert.NotNil(t, sdk.onTerminate) assert.NotNil(t, sdk.onSuspend) + assert.NotNil(t, sdk.onResume) + assert.NotNil(t, sdk.onComplete) } func Test_NewDataPlaneSDK_MissingStore(t *testing.T) { diff --git a/pkg/dsdk/dsdk_test.go b/pkg/dsdk/dsdk_test.go index 81081e3..c8e95e1 100644 --- a/pkg/dsdk/dsdk_test.go +++ b/pkg/dsdk/dsdk_test.go @@ -4,697 +4,561 @@ import ( "context" "errors" "fmt" - "net/url" + "sync" "testing" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) -func Test_DataPlaneSDK_Start(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onStart: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: Started}, nil - }, - } - - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(nil, ErrNotFound) - store.EXPECT().Create(ctx, mock.Anything).Return(nil) - - callbackURL, _ := url.Parse("http://test.com/callback") - - _, _ = dsdk.Start(ctx, DataFlowStartMessage{DataFlowBaseMessage: DataFlowBaseMessage{ - ProcessID: "process123", - AgreementID: "agreement123", - DatasetID: "dataset123", - ParticipantID: "participant123", - DataspaceContext: "dscontext", - CounterPartyID: "counterparty123", - CallbackAddress: CallbackURL(*callbackURL), - TransferType: TransferType{DestinationType: "test", FlowType: Pull}, - }}) -} - -func Test_DataPlaneSDK_Start_VerifySdkCallback(t *testing.T) { +// --- Prepare --- + +func Test_Prepare_Pull(t *testing.T) { + sdk, store := newTestSDK(t, WithPrepareProcessor(func(_ context.Context, flow *DataFlow, _ *DataPlaneSDK, _ *ProcessorOptions) (*DataFlowStatusMessage, error) { + // a pull consumer must not hand out an address; the SDK drops it + return &DataFlowStatusMessage{MessageID: "ignored", DataFlowID: "ignored", State: Prepared, DataAddress: newDataAddress()}, nil + })) + msg := createPrepareMessage(pullProfile) + + response, err := sdk.Prepare(context.Background(), msg) + require.NoError(t, err) + assert.Equal(t, Prepared, response.State) + assert.Equal(t, msg.DataFlowID, response.DataFlowID) + assert.NotEmpty(t, response.MessageID) + assert.NotEqual(t, "ignored", response.MessageID) + assert.Nil(t, response.DataAddress) + + flow := store.get(t, msg.DataFlowID) + assert.True(t, flow.Consumer) + assert.Equal(t, Prepared, flow.State) + assert.Equal(t, msg.Profile, flow.Profile) + assert.Equal(t, msg.Claims, flow.Claims) + assert.Nil(t, flow.DestinationDataAddress) +} + +func Test_Prepare_Push_ReturnsDataAddress(t *testing.T) { + address := newDataAddress() + sdk, store := newTestSDK(t, WithPrepareProcessor(func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowStatusMessage, error) { + return &DataFlowStatusMessage{State: Prepared, DataAddress: address}, nil + })) + msg := createPrepareMessage(pushProfile) + msg.Labels = []string{"gold"} + msg.Metadata = map[string]any{"bucket": "b"} + + response, err := sdk.Prepare(context.Background(), msg) + require.NoError(t, err) + assert.Equal(t, address, response.DataAddress) + + flow := store.get(t, msg.DataFlowID) + assert.Equal(t, address, flow.DestinationDataAddress) + assert.Equal(t, []string{"gold"}, flow.Labels) + assert.Equal(t, map[string]any{"bucket": "b"}, flow.Metadata) +} + +func Test_Prepare_VerifyProcessorState(t *testing.T) { tests := []struct { state DataFlowState expectError bool }{ - { - Preparing, - true, - }, - { - Prepared, - true, - }, - { - Starting, - false, - }, - { - Started, - false, - }, - { - Suspended, - true, - }, - { - Completed, - true, - }, + {Initialized, true}, + {Preparing, false}, + {Prepared, false}, + {Starting, true}, + {Started, true}, + {Suspended, true}, + {Completed, true}, + {Terminated, true}, } - for _, test := range tests { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onStart: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: test.state}, nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(nil, ErrNotFound) - - // create is not invoked if the SDK hook returns an invalid state - if !test.expectError { - store.EXPECT().Create(ctx, mock.MatchedBy(func(df *DataFlow) bool { - return df.State == test.state - })).Return(nil) - } - - _, err := dsdk.Start(ctx, createStartMessage()) - if test.expectError { - // assert.Nil(t, response) - assert.Error(t, err) - } else { - assert.NoError(t, err) - } + t.Run(test.state.String(), func(t *testing.T) { + sdk, store := newTestSDK(t, WithPrepareProcessor(stateProcessor(test.state))) + msg := createPrepareMessage(pullProfile) + response, err := sdk.Prepare(context.Background(), msg) + if test.expectError { + assert.Error(t, err) + assert.Nil(t, response) + assert.False(t, store.has(msg.DataFlowID)) + return + } + require.NoError(t, err) + assert.Equal(t, test.state, store.get(t, msg.DataFlowID).State) + }) + } +} + +func Test_Prepare_Duplicate(t *testing.T) { + var calls []bool + sdk, store := newTestSDK(t, WithPrepareProcessor(func(_ context.Context, _ *DataFlow, _ *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + calls = append(calls, options.Duplicate) + return &DataFlowStatusMessage{State: Prepared}, nil + })) + msg := createPrepareMessage(pullProfile) + _, err := sdk.Prepare(context.Background(), msg) + require.NoError(t, err) + + response, err := sdk.Prepare(context.Background(), msg) + require.NoError(t, err) + assert.Equal(t, Prepared, response.State) + assert.Equal(t, []bool{false, true}, calls) + assert.Equal(t, Prepared, store.get(t, msg.DataFlowID).State) +} + +func Test_Prepare_InWrongState(t *testing.T) { + sdk, store := newTestSDK(t) + msg := createPrepareMessage(pullProfile) + store.put(&DataFlow{ID: msg.DataFlowID, State: Started}) + + _, err := sdk.Prepare(context.Background(), msg) + assert.ErrorIs(t, err, ErrConflict) +} + +func Test_Prepare_ProcessorError(t *testing.T) { + sdk, store := newTestSDK(t, WithPrepareProcessor(func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowStatusMessage, error) { + return nil, errors.New("boom") + })) + msg := createPrepareMessage(pullProfile) + _, err := sdk.Prepare(context.Background(), msg) + assert.ErrorContains(t, err, "boom") + assert.False(t, store.has(msg.DataFlowID)) +} + +func Test_Prepare_ProcessorReturnsNil(t *testing.T) { + sdk, _ := newTestSDK(t, WithPrepareProcessor(func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowStatusMessage, error) { + return nil, nil + })) + _, err := sdk.Prepare(context.Background(), createPrepareMessage(pullProfile)) + assert.ErrorIs(t, err, errNilResponse) +} + +func Test_Prepare_InvalidMessage(t *testing.T) { + sdk, _ := newTestSDK(t) + msg := createPrepareMessage(pullProfile) + msg.Claims = nil + _, err := sdk.Prepare(context.Background(), msg) + assert.ErrorIs(t, err, ErrValidation) +} + +// --- Start --- + +func Test_Start_Pull_ReturnsDataAddress(t *testing.T) { + address := newDataAddress() + var received *ProcessorOptions + sdk, store := newTestSDK(t, WithStartProcessor(func(_ context.Context, _ *DataFlow, _ *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + received = options + return &DataFlowStatusMessage{State: Started, DataAddress: address}, nil + })) + msg := createStartMessage(pullProfile, nil) + + response, err := sdk.Start(context.Background(), msg) + require.NoError(t, err) + assert.Equal(t, Started, response.State) + assert.Equal(t, msg.DataFlowID, response.DataFlowID) + assert.NotEmpty(t, response.MessageID) + assert.Equal(t, address, response.DataAddress) + assert.False(t, received.Duplicate) + assert.Nil(t, received.DataAddress) + + flow := store.get(t, msg.DataFlowID) + assert.False(t, flow.Consumer) + assert.Equal(t, Started, flow.State) + assert.Equal(t, address, flow.SourceDataAddress) +} + +func Test_Start_Push_ReceivesDataAddress(t *testing.T) { + address := newDataAddress() + var received *ProcessorOptions + sdk, store := newTestSDK(t, WithStartProcessor(func(_ context.Context, _ *DataFlow, _ *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + received = options + return &DataFlowStatusMessage{State: Started, DataAddress: newDataAddress()}, nil + })) + msg := createStartMessage(pushProfile, address) + + response, err := sdk.Start(context.Background(), msg) + require.NoError(t, err) + assert.Nil(t, response.DataAddress, "a push provider must not return a data address") + assert.Equal(t, address, received.DataAddress) + assert.Equal(t, address, store.get(t, msg.DataFlowID).DestinationDataAddress) +} + +func Test_Start_DataAddressRules(t *testing.T) { + tests := map[string]struct { + profile string + address *DataAddress + }{ + "pull with data address": {pullProfile, newDataAddress()}, + "push without data address": {pushProfile, nil}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + sdk, store := newTestSDK(t) + msg := createStartMessage(tc.profile, tc.address) + _, err := sdk.Start(context.Background(), msg) + assert.ErrorIs(t, err, ErrInvalidInput) + assert.False(t, store.has(msg.DataFlowID)) + }) } } -func Test_DataPlaneSDK_Start_SdkCallbackInvalidState(t *testing.T) { +func Test_Start_VerifyProcessorState(t *testing.T) { tests := []struct { - initialState DataFlowState - sdkTargetState DataFlowState - expectError bool + state DataFlowState + expectError bool }{ - { - Starting, - Starting, - false, - }, - { - Started, - Started, - false, - }, - { - Prepared, - Started, - false, - }, - { - Prepared, - Prepared, - true, - }} - + {Preparing, true}, + {Prepared, true}, + {Starting, false}, + {Started, false}, + {Suspended, true}, + {Completed, true}, + } for _, test := range tests { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onStart: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: test.sdkTargetState}, nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(&DataFlow{ - State: test.initialState, - Consumer: true, //cover the PREPARED case - }, nil) - - // create is not invoked if the SDK hook returns an invalid initialState - if !test.expectError { - store.EXPECT().Save(ctx, mock.MatchedBy(func(df *DataFlow) bool { - return df.State == test.sdkTargetState - })).Return(nil) - } - - _, err := dsdk.Start(ctx, createStartMessage()) - if test.expectError { - // assert.Nil(t, response) - assert.Error(t, err) - } else { - assert.NoError(t, err) - } + t.Run(test.state.String(), func(t *testing.T) { + sdk, store := newTestSDK(t, WithStartProcessor(stateProcessor(test.state))) + msg := createStartMessage(pullProfile, nil) + _, err := sdk.Start(context.Background(), msg) + if test.expectError { + assert.Error(t, err) + assert.False(t, store.has(msg.DataFlowID)) + return + } + require.NoError(t, err) + assert.Equal(t, test.state, store.get(t, msg.DataFlowID).State) + }) } } -func Test_DataPlaneSDK_Start_AlreadyStarted(t *testing.T) { - states := []DataFlowState{ - Started, - Starting, - } +func Test_Start_ExistingFlow_Conflict(t *testing.T) { + for _, state := range []DataFlowState{Prepared, Starting, Started, Suspended, Completed, Terminated} { + t.Run(state.String(), func(t *testing.T) { + sdk, store := newTestSDK(t) + msg := createStartMessage(pullProfile, nil) + store.put(&DataFlow{ID: msg.DataFlowID, State: state, Consumer: state == Prepared}) - for _, state := range states { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onStart: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: state}, nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(&DataFlow{ - State: state, - }, nil) - store.EXPECT().Save(ctx, mock.AnythingOfType("*dsdk.DataFlow")).Return(nil) - - _, err := dsdk.Start(ctx, createStartMessage()) - assert.NoError(t, err) + _, err := sdk.Start(context.Background(), msg) + assert.ErrorIs(t, err, ErrConflict) + assert.Equal(t, state, store.get(t, msg.DataFlowID).State) + }) } } -func Test_DataPlaneSDK_Start_ConsumerPrepared(t *testing.T) { +func Test_Start_StoreError(t *testing.T) { store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onStart: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: Started}, nil - }, + sdk := newMockSDK(store) + msg := createStartMessage(pullProfile, nil) + store.EXPECT().FindById(mock.Anything, msg.DataFlowID).Return(nil, errors.New("db down")) + + _, err := sdk.Start(context.Background(), msg) + assert.ErrorContains(t, err, "db down") +} + +// --- Started --- + +func Test_Started_Pull(t *testing.T) { + address := newDataAddress() + var received *ProcessorOptions + sdk, store := newTestSDK(t, WithStartProcessor(func(_ context.Context, _ *DataFlow, _ *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + received = options + return &DataFlowStatusMessage{State: Started}, nil + })) + store.put(consumerFlow("flow", pullProfile, Prepared)) + + response, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m", DataAddress: address}) + require.NoError(t, err) + assert.Equal(t, Started, response.State) + assert.Equal(t, "flow", response.DataFlowID) + assert.NotEmpty(t, response.MessageID) + assert.Nil(t, response.DataAddress) + assert.Equal(t, address, received.DataAddress) + assert.False(t, received.Duplicate) + + flow := store.get(t, "flow") + assert.Equal(t, Started, flow.State) + assert.Equal(t, address, flow.SourceDataAddress) +} + +func Test_Started_Push(t *testing.T) { + sdk, store := newTestSDK(t) + flow := consumerFlow("flow", pushProfile, Prepared) + flow.DestinationDataAddress = newDataAddress() + store.put(flow) + + response, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m"}) + require.NoError(t, err) + assert.Equal(t, Started, response.State) + assert.Nil(t, response.DataAddress) +} + +func Test_Started_DataAddressRules(t *testing.T) { + tests := map[string]struct { + profile string + address *DataAddress + }{ + "pull without data address": {pullProfile, nil}, + "push with data address": {pushProfile, newDataAddress()}, } - - ctx := context.Background() - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(&DataFlow{ - State: Prepared, - Consumer: true, - }, nil) - store.EXPECT().Save(ctx, mock.AnythingOfType("*dsdk.DataFlow")).Return(nil) - - _, err := dsdk.Start(ctx, createStartMessage()) - assert.NoError(t, err) -} - -func Test_DataPlaneSDK_StartById_Exists(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onStart: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: Started}, nil - }, + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(consumerFlow("flow", tc.profile, Prepared)) + _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m", DataAddress: tc.address}) + assert.ErrorIs(t, err, ErrInvalidInput) + assert.Equal(t, Prepared, store.get(t, "flow").State) + }) } - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(&DataFlow{ - ID: "process123", - State: Prepared, - Consumer: true, - }, nil) - store.EXPECT().Save(ctx, mock.AnythingOfType("*dsdk.DataFlow")).Return(nil) - - r, err := dsdk.StartById(ctx, "process123", createStartByIdMessage()) - assert.NoError(t, err) - assert.Equal(t, r.State, Started) } -func Test_DataPlaneSDK_StartById_NotExists(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - } - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(nil, ErrNotFound) +func Test_Started_Duplicate(t *testing.T) { + var received *ProcessorOptions + sdk, store := newTestSDK(t, WithStartProcessor(func(_ context.Context, _ *DataFlow, _ *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + received = options + return &DataFlowStatusMessage{State: Started}, nil + })) + store.put(consumerFlow("flow", pushProfile, Started)) - _, err := dsdk.StartById(ctx, "process123", createStartByIdMessage()) - assert.ErrorIs(t, err, ErrNotFound) + _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m"}) + require.NoError(t, err) + assert.True(t, received.Duplicate) } -func Test_DataPlaneSDK_StartById_NotConsumer(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, +func Test_Started_WrongState(t *testing.T) { + for _, state := range []DataFlowState{Initialized, Preparing, Suspended, Completed, Terminated} { + t.Run(state.String(), func(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(consumerFlow("flow", pushProfile, state)) + _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m"}) + assert.ErrorIs(t, err, ErrInvalidTransition) + }) } - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(&DataFlow{ - ID: "process123", - State: Prepared, - Consumer: false, - }, nil) +} - r, err := dsdk.StartById(ctx, "process123", createStartByIdMessage()) +func Test_Started_NotConsumer(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", Profile: pushProfile, State: Prepared}) + _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m"}) assert.ErrorIs(t, err, ErrInvalidInput) - assert.Nil(t, r) } -func Test_DataPlaneSDK_StartById_WrongState(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - } - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(&DataFlow{ - ID: "process123", - State: Uninitialized, - Consumer: true, - }, nil) - - r, err := dsdk.StartById(ctx, "process123", createStartByIdMessage()) - assert.ErrorIs(t, err, ErrInvalidTransition) - assert.Nil(t, r) +func Test_Started_NotFound(t *testing.T) { + sdk, _ := newTestSDK(t) + _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m"}) + assert.ErrorIs(t, err, ErrNotFound) } -func Test_DataPlaneSDK_StartById_AlreadyStarted(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onStart: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: Started}, nil - }, - } - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(&DataFlow{ - ID: "process123", - State: Started, - Consumer: true, - }, nil) - store.EXPECT().Save(ctx, mock.AnythingOfType("*dsdk.DataFlow")).Return(nil) - - r, err := dsdk.StartById(ctx, "process123", createStartByIdMessage()) - assert.NoError(t, err) - assert.Equal(t, r.State, Started) +func Test_Started_RequiresMessageID(t *testing.T) { + sdk, _ := newTestSDK(t) + _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{}) + assert.ErrorIs(t, err, ErrValidation) } -func Test_DataPlaneSDK_Status(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - } +// --- Suspend / Resume --- - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(&DataFlow{ - ID: "process123", - State: Preparing, - }, nil) +func Test_Suspend(t *testing.T) { + var suspended bool + sdk, store := newTestSDK(t, WithSuspendProcessor(func(context.Context, *DataFlow) error { + suspended = true + return nil + })) + store.put(&DataFlow{ID: "flow", Profile: pullProfile, State: Started}) - flow, err := dsdk.Status(ctx, "process123") - assert.NoError(t, err) - assert.Equal(t, flow.State, Preparing) + require.NoError(t, sdk.Suspend(context.Background(), "flow", DataFlowSuspendMessage{MessageID: "m", Reason: "maintenance"})) + assert.True(t, suspended) + flow := store.get(t, "flow") + assert.Equal(t, Suspended, flow.State) + assert.Equal(t, SuspendedByRemote, flow.SuspendedBy) + assert.Equal(t, "maintenance", flow.ErrorDetail) } -func Test_DataPlaneSDK_Status_WhenNotFound(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - } - - ctx := context.Background() - store.EXPECT().FindById(ctx, "process123").Return(nil, ErrNotFound) - - flow, err := dsdk.Status(ctx, "process123") - assert.ErrorIs(t, err, ErrNotFound) - assert.Nil(t, flow) +func Test_Suspend_AlreadySuspended(t *testing.T) { + sdk, store := newTestSDK(t, WithSuspendProcessor(func(context.Context, *DataFlow) error { + t.Fatal("processor must not be invoked for a duplicate") + return nil + })) + store.put(&DataFlow{ID: "flow", State: Suspended, SuspendedBy: SuspendedByRemote}) + assert.NoError(t, sdk.Suspend(context.Background(), "flow", DataFlowSuspendMessage{MessageID: "m"})) } -func Test_DataPlaneSDK_Prepare(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onPrepare: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: Prepared}, nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(nil, ErrNotFound) - store.EXPECT().Create(ctx, mock.AnythingOfType("*dsdk.DataFlow")).Return(nil) - - response, err := dsdk.Prepare(ctx, createPrepareMessage()) - assert.NoError(t, err) - assert.NotNil(t, response) +func Test_Suspend_WrongState(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", State: Prepared}) + assert.ErrorIs(t, sdk.Suspend(context.Background(), "flow", DataFlowSuspendMessage{MessageID: "m"}), ErrInvalidTransition) } -func Test_DataPlaneSDK_Prepare_VerifySdkCallback(t *testing.T) { - tests := []struct { - state DataFlowState - expectError bool - }{ - { - Preparing, - false, - }, - { - Prepared, - false, - }, - { - Starting, - true, - }, - { - Started, - true, - }, - { - Completed, - true, - }, - } - for _, test := range tests { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onPrepare: func(context.Context, *DataFlow, *DataPlaneSDK, *ProcessorOptions) (*DataFlowResponseMessage, error) { - return &DataFlowResponseMessage{State: test.state}, nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(nil, ErrNotFound) - - // create is not invoked if the SDK hook returns an invalid initialState - if !test.expectError { - store.EXPECT().Create(ctx, mock.MatchedBy(func(df *DataFlow) bool { - return df.State == test.state - })).Return(nil) - } - - _, err := dsdk.Prepare(ctx, createPrepareMessage()) - if test.expectError { - // assert.Nil(t, response) - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - } +func Test_Suspend_ProcessorError(t *testing.T) { + sdk, store := newTestSDK(t, WithSuspendProcessor(func(context.Context, *DataFlow) error { + return errors.New("boom") + })) + store.put(&DataFlow{ID: "flow", State: Started}) + assert.ErrorContains(t, sdk.Suspend(context.Background(), "flow", DataFlowSuspendMessage{MessageID: "m"}), "boom") + assert.Equal(t, Started, store.get(t, "flow").State) } -func Test_DataPlaneSDK_Prepare_AlreadyPreparing(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onPrepare: func(ctx context.Context, flow *DataFlow, sdk *DataPlaneSDK, opts *ProcessorOptions) (*DataFlowResponseMessage, error) { - assert.Equal(t, opts.Duplicate, true) - return &DataFlowResponseMessage{State: Prepared}, nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(&DataFlow{ - ID: "process123", - State: Preparing, - }, nil) - store.EXPECT().Save(ctx, mock.AnythingOfType("*dsdk.DataFlow")).Return(nil) - - response, err := dsdk.Prepare(ctx, createPrepareMessage()) - assert.NoError(t, err) - assert.NotNil(t, response) +func Test_Suspend_NotFound(t *testing.T) { + sdk, _ := newTestSDK(t) + assert.ErrorIs(t, sdk.Suspend(context.Background(), "flow", DataFlowSuspendMessage{MessageID: "m"}), ErrNotFound) } -func Test_DataPlaneSDK_Prepare_InWrongState(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - } - - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(&DataFlow{ - ID: "process123", - State: Started, - }, nil) - store.AssertNotCalled(t, "Create", mock.AnythingOfType("*dsdk.DataFlow")) - - _, err := dsdk.Prepare(ctx, createPrepareMessage()) - assert.ErrorIs(t, err, ErrConflict) - assert.ErrorContains(t, err, "is not in PREPARING or PREPARED state") +func Test_Suspend_RequiresMessageID(t *testing.T) { + sdk, _ := newTestSDK(t) + assert.ErrorIs(t, sdk.Suspend(context.Background(), "flow", DataFlowSuspendMessage{}), ErrValidation) } -func Test_DataPlaneSDK_Prepare_SdkCallbackReturnsError(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onPrepare: func(ctx context.Context, flow *DataFlow, sdk *DataPlaneSDK, opts *ProcessorOptions) (*DataFlowResponseMessage, error) { - return nil, errors.New("some error") - }, - } +func Test_Resume_ProviderPull_ReturnsDataAddress(t *testing.T) { + address := newDataAddress() + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", Profile: pullProfile, State: Suspended, SuspendedBy: SuspendedByRemote, SourceDataAddress: address}) - ctx := context.Background() - - store.EXPECT().FindById(mock.Anything, mock.AnythingOfType("string")).Return(nil, ErrNotFound) - store.AssertNotCalled(t, "Create", mock.AnythingOfType("*dsdk.DataFlow")) + response, err := sdk.Resume(context.Background(), "flow", DataFlowResumeMessage{MessageID: "m"}) + require.NoError(t, err) + assert.Equal(t, Started, response.State) + assert.Equal(t, "flow", response.DataFlowID) + assert.Equal(t, address, response.DataAddress) - _, err := dsdk.Prepare(ctx, createPrepareMessage()) - assert.ErrorContains(t, err, "processing data flow process123: some error") + flow := store.get(t, "flow") + assert.Equal(t, Started, flow.State) + assert.Equal(t, SuspendedByNone, flow.SuspendedBy) } -func Test_DataPlaneSDK_Terminate(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onTerminate: func(ctx context.Context, flow *DataFlow) error { - return nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Started, - }, nil) +func Test_Resume_ProviderPush_ReceivesDataAddress(t *testing.T) { + address := newDataAddress() + var received *ProcessorOptions + sdk, store := newTestSDK(t, WithResumeProcessor(func(_ context.Context, _ *DataFlow, _ *DataPlaneSDK, options *ProcessorOptions) (*DataFlowStatusMessage, error) { + received = options + return &DataFlowStatusMessage{State: Started}, nil + })) + store.put(&DataFlow{ID: "flow", Profile: pushProfile, State: Suspended, SuspendedBy: SuspendedByRemote}) - store.EXPECT().Save(ctx, mock.MatchedBy(func(df *DataFlow) bool { - return df.State == Terminated - })).Return(nil) + response, err := sdk.Resume(context.Background(), "flow", DataFlowResumeMessage{MessageID: "m", DataAddress: address}) + require.NoError(t, err) + assert.Nil(t, response.DataAddress) + assert.Equal(t, address, received.DataAddress) + assert.Equal(t, address, store.get(t, "flow").DestinationDataAddress) +} - err := dsdk.Terminate(ctx, "flow123", "") +func Test_Resume_SuspendedLocally(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", Profile: pullProfile, State: Suspended, SuspendedBy: SuspendedByLocal}) - assert.NoError(t, err) + _, err := sdk.Resume(context.Background(), "flow", DataFlowResumeMessage{MessageID: "m"}) + assert.ErrorIs(t, err, ErrInvalidTransition) + assert.Equal(t, Suspended, store.get(t, "flow").State) } -func Test_DataPlaneSDK_Terminate_NotFound(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onTerminate: func(ctx context.Context, flow *DataFlow) error { - return nil - }, +func Test_Resume_WrongState(t *testing.T) { + for _, state := range []DataFlowState{Prepared, Started, Completed, Terminated} { + t.Run(state.String(), func(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", Profile: pullProfile, State: state}) + _, err := sdk.Resume(context.Background(), "flow", DataFlowResumeMessage{MessageID: "m"}) + assert.ErrorIs(t, err, ErrInvalidTransition) + }) } +} - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(nil, ErrNotFound) - err := dsdk.Terminate(ctx, "flow123", "") +func Test_Resume_ProcessorInvalidState(t *testing.T) { + sdk, store := newTestSDK(t, WithResumeProcessor(stateProcessor(Suspended))) + store.put(&DataFlow{ID: "flow", Profile: pullProfile, State: Suspended, SuspendedBy: SuspendedByRemote}) + _, err := sdk.Resume(context.Background(), "flow", DataFlowResumeMessage{MessageID: "m"}) + assert.Error(t, err) + assert.Equal(t, Suspended, store.get(t, "flow").State) +} - assert.ErrorContains(t, err, "not found") +func Test_SuspendThenResume(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", Profile: pullProfile, State: Started}) + require.NoError(t, sdk.Suspend(context.Background(), "flow", DataFlowSuspendMessage{MessageID: "m1"})) + _, err := sdk.Resume(context.Background(), "flow", DataFlowResumeMessage{MessageID: "m2"}) + require.NoError(t, err) + assert.Equal(t, Started, store.get(t, "flow").State) } -func Test_DataPlaneSDK_Terminate_AlreadyTerminated(t *testing.T) { +// --- Terminate --- - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onTerminate: func(ctx context.Context, flow *DataFlow) error { - return nil - }, +func Test_Terminate(t *testing.T) { + for _, state := range []DataFlowState{Initialized, Preparing, Prepared, Starting, Started, Suspended} { + t.Run(state.String(), func(t *testing.T) { + var terminated bool + sdk, store := newTestSDK(t, WithTerminateProcessor(func(context.Context, *DataFlow) error { + terminated = true + return nil + })) + store.put(&DataFlow{ID: "flow", State: state}) + require.NoError(t, sdk.Terminate(context.Background(), "flow", DataFlowTerminateMessage{MessageID: "m", Reason: "violation"})) + assert.True(t, terminated) + flow := store.get(t, "flow") + assert.Equal(t, Terminated, flow.State) + assert.Equal(t, "violation", flow.ErrorDetail) + }) } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Terminated, // already terminated - }, nil) - - // no transition and no save call expected - - err := dsdk.Terminate(ctx, "flow123", "") - - assert.NoError(t, err) } -func Test_DataPlaneSDK_Terminate_SdkCallbackError(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onTerminate: func(ctx context.Context, flow *DataFlow) error { - return fmt.Errorf("some error") - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Started, - }, nil) - - err := dsdk.Terminate(ctx, "flow123", "") - - assert.ErrorContains(t, err, "some error") +func Test_Terminate_Completed(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", State: Completed}) + assert.ErrorIs(t, sdk.Terminate(context.Background(), "flow", DataFlowTerminateMessage{MessageID: "m"}), ErrInvalidTransition) + assert.Equal(t, Completed, store.get(t, "flow").State) } -func Test_DataPlaneSDK_Suspend(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onSuspend: func(ctx context.Context, flow *DataFlow) error { - return nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Started, - }, nil) - - store.EXPECT().Save(ctx, mock.MatchedBy(func(df *DataFlow) bool { - return df.State == Suspended - })).Return(nil) - - err := dsdk.Suspend(ctx, "flow123", "") - - assert.NoError(t, err) +func Test_Terminate_AlreadyTerminated(t *testing.T) { + sdk, store := newTestSDK(t, WithTerminateProcessor(func(context.Context, *DataFlow) error { + t.Fatal("processor must not be invoked for a duplicate") + return nil + })) + store.put(&DataFlow{ID: "flow", State: Terminated}) + assert.NoError(t, sdk.Terminate(context.Background(), "flow", DataFlowTerminateMessage{MessageID: "m"})) } -func Test_DataPlaneSDK_Suspend_NotFound(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onSuspend: func(ctx context.Context, flow *DataFlow) error { - return nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(nil, ErrNotFound) - err := dsdk.Suspend(ctx, "flow123", "") - - assert.ErrorContains(t, err, "not found") +func Test_Terminate_ProcessorError(t *testing.T) { + sdk, store := newTestSDK(t, WithTerminateProcessor(func(context.Context, *DataFlow) error { + return errors.New("boom") + })) + store.put(&DataFlow{ID: "flow", State: Started}) + assert.ErrorContains(t, sdk.Terminate(context.Background(), "flow", DataFlowTerminateMessage{MessageID: "m"}), "boom") + assert.Equal(t, Started, store.get(t, "flow").State) } -func Test_DataPlaneSDK_Suspend_AlreadySuspended(t *testing.T) { - - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onTerminate: func(ctx context.Context, flow *DataFlow) error { - return nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Suspended, // already suspended - }, nil) - - // no transition and no save call expected - - err := dsdk.Suspend(ctx, "flow123", "") - - assert.NoError(t, err) +func Test_Terminate_NotFound(t *testing.T) { + sdk, _ := newTestSDK(t) + assert.ErrorIs(t, sdk.Terminate(context.Background(), "flow", DataFlowTerminateMessage{MessageID: "m"}), ErrNotFound) } -func Test_DataPlaneSDK_Suspend_SdkCallbackError(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onSuspend: func(ctx context.Context, flow *DataFlow) error { - return fmt.Errorf("some error") - }, - } - - ctx := context.Background() +func Test_Terminate_RequiresMessageID(t *testing.T) { + sdk, _ := newTestSDK(t) + assert.ErrorIs(t, sdk.Terminate(context.Background(), "flow", DataFlowTerminateMessage{}), ErrValidation) +} - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Started, - }, nil) +// --- Status --- - err := dsdk.Suspend(ctx, "flow123", "") +func Test_Status(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow", State: Started}) + flow, err := sdk.Status(context.Background(), "flow") + require.NoError(t, err) + assert.Equal(t, Started, flow.State) - assert.ErrorContains(t, err, "some error") + _, err = sdk.Status(context.Background(), "unknown") + assert.ErrorIs(t, err, ErrNotFound) } -func Test_DataPlaneSDK_Completed(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onComplete: func(ctx context.Context, flow *DataFlow) error { - return nil - }, - } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Started, // already suspended - }, nil) - store.EXPECT().Save(ctx, mock.MatchedBy(func(df *DataFlow) bool { - return df.State == Completed - })).Return(nil) +// --- Complete --- - err := dsdk.Complete(ctx, "flow123") +func Test_DataPlaneSDK_Completed(t *testing.T) { + var completed bool + sdk, store := newTestSDK(t, WithCompleteProcessor(func(context.Context, *DataFlow) error { + completed = true + return nil + })) + store.put(&DataFlow{ID: "flow123", State: Started}) - assert.NoError(t, err) + require.NoError(t, sdk.Complete(context.Background(), "flow123")) + assert.True(t, completed) + assert.Equal(t, Completed, store.get(t, "flow123").State) } func Test_DataPlaneSDK_Completed_StoreError(t *testing.T) { store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onComplete: func(ctx context.Context, flow *DataFlow) error { - return nil - }, - } - + sdk := newMockSDK(store) ctx := context.Background() store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ @@ -703,122 +567,138 @@ func Test_DataPlaneSDK_Completed_StoreError(t *testing.T) { }, nil) store.EXPECT().Save(ctx, mock.AnythingOfType("*dsdk.DataFlow")).Return(fmt.Errorf("store error")) - err := dsdk.Complete(ctx, "flow123") + err := sdk.Complete(ctx, "flow123") assert.ErrorContains(t, err, "store error") } func Test_DataPlaneSDK_Completed_AlreadyCompleted(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - } - - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Completed, // already suspended - }, nil) - - err := dsdk.Complete(ctx, "flow123") + sdk, store := newTestSDK(t, WithCompleteProcessor(func(context.Context, *DataFlow) error { + t.Fatal("processor must not be invoked for a duplicate") + return nil + })) + store.put(&DataFlow{ID: "flow123", State: Completed}) + assert.NoError(t, sdk.Complete(context.Background(), "flow123")) +} - assert.NoError(t, err) +func Test_DataPlaneSDK_Completed_ProcessorError(t *testing.T) { + sdk, store := newTestSDK(t, WithCompleteProcessor(func(context.Context, *DataFlow) error { + return fmt.Errorf("some error") + })) + store.put(&DataFlow{ID: "flow123", State: Started}) + assert.ErrorContains(t, sdk.Complete(context.Background(), "flow123"), "some error") + assert.Equal(t, Started, store.get(t, "flow123").State) } -func Test_DataPlaneSDK_Completed_SdkError(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onComplete: func(ctx context.Context, flow *DataFlow) error { - return fmt.Errorf("some error") - }, - } +func Test_DataPlaneSDK_Completed_WrongState(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(&DataFlow{ID: "flow123", State: Terminated}) + assert.ErrorIs(t, sdk.Complete(context.Background(), "flow123"), ErrInvalidTransition) +} - ctx := context.Background() +func Test_DataPlaneSDK_Completed_NotFound(t *testing.T) { + sdk, _ := newTestSDK(t) + assert.ErrorIs(t, sdk.Complete(context.Background(), "flow123"), ErrNotFound) +} - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Started, // already suspended - }, nil) +func Test_DataPlaneSDK_Completed_EmptyID(t *testing.T) { + sdk, _ := newTestSDK(t) + assert.ErrorIs(t, sdk.Complete(context.Background(), ""), ErrInvalidInput) +} - err := dsdk.Complete(ctx, "flow123") +// --- helpers --- - assert.ErrorContains(t, err, "some error") +func newTestSDK(t *testing.T, options ...DataPlaneSDKOption) (*DataPlaneSDK, *fakeStore) { + t.Helper() + store := &fakeStore{flows: map[string]*DataFlow{}} + sdk, err := NewDataPlaneSDK(append([]DataPlaneSDKOption{WithStore(store), WithTransactionContext(&mockTrxContext{})}, options...)...) + require.NoError(t, err) + return sdk, store } -func Test_DataPlaneSDK_Completed_WrongState(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onComplete: func(ctx context.Context, flow *DataFlow) error { - return nil - }, - } +func newMockSDK(store DataplaneStore) *DataPlaneSDK { + sdk, _ := NewDataPlaneSDK(WithStore(store), WithTransactionContext(&mockTrxContext{})) + return sdk +} - ctx := context.Background() +func consumerFlow(id string, profile string, state DataFlowState) *DataFlow { + return &DataFlow{ID: id, Consumer: true, Profile: profile, State: state} +} - store.EXPECT().FindById(ctx, "flow123").Return(&DataFlow{ - ID: "flow123", - State: Terminated, - }, nil) +func createPrepareMessage(profile string) DataFlowPrepareMessage { + base := newBaseMessage() + base.Profile = profile + return DataFlowPrepareMessage{DataFlowBaseMessage: base} +} - err := dsdk.Complete(ctx, "flow123") +func createStartMessage(profile string, address *DataAddress) DataFlowStartMessage { + base := newBaseMessage() + base.Profile = profile + base.DataFlowID = uuid.NewString() + return DataFlowStartMessage{DataFlowBaseMessage: base, DataAddress: address} +} - assert.ErrorIs(t, err, ErrInvalidTransition) +// fakeStore is a minimal DataplaneStore that stores copies of the flows. +type fakeStore struct { + mu sync.Mutex + flows map[string]*DataFlow } -func Test_DataPlaneSDK_NotFound(t *testing.T) { - store := NewMockDataplaneStore(t) - dsdk := DataPlaneSDK{ - Store: store, - TrxContext: &mockTrxContext{}, - onComplete: func(ctx context.Context, flow *DataFlow) error { - return nil - }, +func (s *fakeStore) FindById(_ context.Context, id string) (*DataFlow, error) { + s.mu.Lock() + defer s.mu.Unlock() + flow, ok := s.flows[id] + if !ok { + return nil, ErrNotFound } + return flow.Copy(), nil +} - ctx := context.Background() - - store.EXPECT().FindById(ctx, "flow123").Return(nil, ErrNotFound) - - err := dsdk.Complete(ctx, "flow123") +func (s *fakeStore) Create(_ context.Context, flow *DataFlow) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.flows[flow.ID]; ok { + return ErrConflict + } + s.flows[flow.ID] = flow.Copy() + return nil +} - assert.ErrorIs(t, err, ErrNotFound) +func (s *fakeStore) Save(_ context.Context, flow *DataFlow) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.flows[flow.ID]; !ok { + return ErrNotFound + } + s.flows[flow.ID] = flow.Copy() + return nil } -func createPrepareMessage() DataFlowPrepareMessage { - return DataFlowPrepareMessage{DataFlowBaseMessage: createBaseMessage()} +func (s *fakeStore) Delete(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.flows, id) + return nil } -func createStartMessage() DataFlowStartMessage { - return DataFlowStartMessage{ - DataFlowBaseMessage: createBaseMessage(), - } + +func (s *fakeStore) put(flow *DataFlow) { + s.mu.Lock() + defer s.mu.Unlock() + s.flows[flow.ID] = flow.Copy() } -func createBaseMessage() DataFlowBaseMessage { - return DataFlowBaseMessage{ - MessageID: uuid.New().String(), - ParticipantID: uuid.New().String(), - CounterPartyID: uuid.New().String(), - DataspaceContext: uuid.New().String(), - ProcessID: "process123", - AgreementID: uuid.New().String(), - DatasetID: uuid.New().String(), - CallbackAddress: CallbackURL{Scheme: "http", Host: "test.com", Path: "/callback"}, - TransferType: TransferType{ - DestinationType: "test-type", - FlowType: Pull, - }, - DataAddress: &DataAddress{}, - } + +func (s *fakeStore) has(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.flows[id] + return ok } -func createStartByIdMessage() DataFlowStartedNotificationMessage { - return DataFlowStartedNotificationMessage{ - DataAddress: &DataAddress{}, - } + +func (s *fakeStore) get(t *testing.T, id string) *DataFlow { + t.Helper() + flow, err := s.FindById(context.Background(), id) + require.NoError(t, err) + return flow } type mockTrxContext struct { diff --git a/pkg/dsdk/messages.go b/pkg/dsdk/messages.go index be648c3..2ae5256 100644 --- a/pkg/dsdk/messages.go +++ b/pkg/dsdk/messages.go @@ -6,82 +6,106 @@ import ( var v = validator.New() +// DataFlowBaseMessage contains the properties shared by the DataFlowPrepareMessage and the DataFlowStartMessage. type DataFlowBaseMessage struct { - MessageID string `json:"messageID" validate:"required"` - ParticipantID string `json:"participantID" validate:"required"` - CounterPartyID string `json:"counterPartyID" validate:"required"` - DataspaceContext string `json:"dataspaceContext" validate:"required"` - ProcessID string `json:"processID" validate:"required"` - AgreementID string `json:"agreementID" validate:"required"` - DatasetID string `json:"datasetID"` - CallbackAddress CallbackURL `json:"callbackAddress" validate:"required,callback-url"` - TransferType TransferType `json:"transferType" validate:"required"` - DataAddress *DataAddress `json:"dataAddress"` -} - -func (d *DataFlowBaseMessage) Validate() error { - err := v.RegisterValidation("callback-url", func(fl validator.FieldLevel) bool { - u, ok := fl.Field().Interface().(CallbackURL) - return ok && !u.IsEmpty() - }) - if err != nil { - return err - } + MessageID string `json:"messageId" validate:"required"` + ParticipantID string `json:"participantId" validate:"required"` + CounterPartyID string `json:"counterPartyId" validate:"required"` + DataspaceContext string `json:"dataspaceContext" validate:"required"` + DataFlowID string `json:"dataFlowId" validate:"required"` + AgreementID string `json:"agreementId" validate:"required"` + DatasetID string `json:"datasetId" validate:"required"` + Profile string `json:"profile" validate:"required"` + Claims map[string]any `json:"claims" validate:"required"` + Labels []string `json:"labels,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} - if err := v.Struct(d); err != nil { - return WrapValidationError(err) - } - return nil +// DataFlowPrepareMessage is sent by the consumer control plane to prepare a data flow. +type DataFlowPrepareMessage struct { + DataFlowBaseMessage +} + +func (d *DataFlowPrepareMessage) Validate() error { + return validate(d) } +// DataFlowStartMessage is sent by the provider control plane to start a data flow. For provider-push transfers it +// carries the DataAddress the data must be pushed to; it must be omitted for consumer-pull transfers. type DataFlowStartMessage struct { DataFlowBaseMessage + DataAddress *DataAddress `json:"dataAddress,omitempty"` } func (d *DataFlowStartMessage) Validate() error { - err := d.DataFlowBaseMessage.Validate() // call base validator - if err != nil { - return WrapValidationError(err) - } - err = v.Struct(d) - if err != nil { - return WrapValidationError(err) - } - return nil + return validate(d) } +// DataFlowStartedNotificationMessage notifies the consumer data plane that a transfer has started. For consumer-pull +// transfers it carries the DataAddress to obtain the data from; it must be omitted for provider-push transfers. type DataFlowStartedNotificationMessage struct { + MessageID string `json:"messageId" validate:"required"` DataAddress *DataAddress `json:"dataAddress,omitempty"` } func (d *DataFlowStartedNotificationMessage) Validate() error { - err := v.Struct(d) - if err != nil { - return WrapValidationError(err) - } - return nil + return validate(d) } -type DataFlowPrepareMessage struct { - DataFlowBaseMessage +// DataFlowSuspendMessage signals the data plane to suspend a data flow. +type DataFlowSuspendMessage struct { + MessageID string `json:"messageId" validate:"required"` + Reason string `json:"reason,omitempty"` } -type DataFlowTransitionMessage struct { - Reason string `json:"reason"` +func (d *DataFlowSuspendMessage) Validate() error { + return validate(d) } -func (d *DataFlowTransitionMessage) Validate() error { - return nil // no special behaviour yet +// DataFlowResumeMessage signals the data plane to resume a suspended data flow. +type DataFlowResumeMessage struct { + MessageID string `json:"messageId" validate:"required"` + DataAddress *DataAddress `json:"dataAddress,omitempty"` } -type DataFlowResponseMessage struct { - DataplaneID string `json:"dataplaneID"` - DataAddress *DataAddress `json:"dataAddress,omitempty"` +func (d *DataFlowResumeMessage) Validate() error { + return validate(d) +} + +// DataFlowTerminateMessage signals the data plane to terminate a data flow. +type DataFlowTerminateMessage struct { + MessageID string `json:"messageId" validate:"required"` + Reason string `json:"reason,omitempty"` +} + +func (d *DataFlowTerminateMessage) Validate() error { + return validate(d) +} + +// DataFlowStatusMessage is returned by the data plane in response to prepare, start, started, resume and status +// requests. +type DataFlowStatusMessage struct { + MessageID string `json:"messageId"` + DataFlowID string `json:"dataFlowId"` State DataFlowState `json:"state"` - Error string `json:"error"` + DataAddress *DataAddress `json:"dataAddress,omitempty"` + Error string `json:"error,omitempty"` } +// DataFlowStatusResponseMessage is returned by the data plane in response to a status request. type DataFlowStatusResponseMessage struct { + DataFlowID string `json:"dataFlowId"` State DataFlowState `json:"state"` - DataFlowID string `json:"dataFlowID"` +} + +// ErrorResponse is the body the data plane returns with HTTP 4xx and 5xx responses. +type ErrorResponse struct { + Error string `json:"error"` +} + +func validate(s any) error { + if err := v.Struct(s); err != nil { + return WrapValidationError(err) + } + return nil } diff --git a/pkg/dsdk/messages_test.go b/pkg/dsdk/messages_test.go index a353695..16ee593 100644 --- a/pkg/dsdk/messages_test.go +++ b/pkg/dsdk/messages_test.go @@ -1,84 +1,259 @@ package dsdk import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" "testing" "github.com/google/uuid" + "github.com/santhosh-tekuri/jsonschema/v6" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func Test_Message_MissingProperties(t *testing.T) { +const ( + signalingSchemaBase = "https://w3id.org/dspace-sig/v1.0/" + dspDataAddressURL = "https://w3id.org/dspace/2025/1/transfer/data-address-schema.json" + pullProfile = "https://w3id.org/dspace-sig/profile/http-pull" + pushProfile = "https://w3id.org/dspace-sig/profile/s3-push" +) + +// dataFlowSchemas are the schemas of the data plane endpoints. The registration and agreement schemas in testdata +// reference further DSP schemas and are not compiled until the SDK implements those endpoints. +var dataFlowSchemas = []string{ + "DataFlowBaseMessage", + "DataFlowPrepareMessage", + "DataFlowStartMessage", + "DataFlowStartedNotificationMessage", + "DataFlowSuspendMessage", + "DataFlowResumeMessage", + "DataFlowTerminateMessage", + "DataFlowStatusMessage", + "DataFlowStatusResponseMessage", +} + +// loadSchemas compiles the published Data Plane Signaling data flow schemas (copied from eclipse-dataplane-signaling +// at a0752cd) and the DSP DataAddress schema they reference, without network access. +func loadSchemas(t *testing.T) map[string]*jsonschema.Schema { + t.Helper() + compiler := jsonschema.NewCompiler() + files, err := filepath.Glob("testdata/schemas/*.json") + require.NoError(t, err) + require.NotEmpty(t, files) - msg := DataFlowBaseMessage{ - MessageID: "test-id", + for _, file := range files { + data, err := os.ReadFile(file) + require.NoError(t, err) + doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + require.NoError(t, err) + + base := filepath.Base(file) + url := signalingSchemaBase + base + if base == "dsp-data-address-schema.json" { + url = dspDataAddressURL + } + require.NoError(t, compiler.AddResource(url, doc)) + } + + schemas := make(map[string]*jsonschema.Schema) + for _, name := range dataFlowSchemas { + schema, err := compiler.Compile(signalingSchemaBase + name + ".schema.json") + require.NoError(t, err, name) + schemas[name] = schema } - err := msg.Validate() - assert.ErrorIs(t, err, ErrValidation) + return schemas } -func Test_Message_Success(t *testing.T) { - msg := newBaseMessage() - err := msg.Validate() - assert.NoError(t, err) +func assertConforms(t *testing.T, schemas map[string]*jsonschema.Schema, schemaName string, message any) { + t.Helper() + schema, ok := schemas[schemaName] + require.True(t, ok, "schema %s not found", schemaName) + data, err := json.Marshal(message) + require.NoError(t, err) + instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + require.NoError(t, err) + assert.NoError(t, schema.Validate(instance), "%s does not conform: %s", schemaName, data) } -func Test_Message_InvalidCallbackAddress(t *testing.T) { - payload := newBaseMessage() - payload.CallbackAddress = CallbackURL{} - err := payload.Validate() +func Test_Messages_ConformToSchemas(t *testing.T) { + schemas := loadSchemas(t) + assert.Len(t, schemas, len(dataFlowSchemas)) + + address := newDataAddress() + base := newBaseMessage() + base.Labels = []string{"gold", "blue"} + base.Metadata = map[string]any{"region": "westeurope"} - assert.ErrorIs(t, err, ErrValidation) + tests := []struct { + schema string + message any + }{ + {"DataFlowPrepareMessage", DataFlowPrepareMessage{DataFlowBaseMessage: base}}, + {"DataFlowPrepareMessage", DataFlowPrepareMessage{DataFlowBaseMessage: newBaseMessage()}}, + {"DataFlowStartMessage", DataFlowStartMessage{DataFlowBaseMessage: base, DataAddress: address}}, + {"DataFlowStartMessage", DataFlowStartMessage{DataFlowBaseMessage: newBaseMessage()}}, + {"DataFlowStartedNotificationMessage", DataFlowStartedNotificationMessage{MessageID: uuid.NewString(), DataAddress: address}}, + {"DataFlowStartedNotificationMessage", DataFlowStartedNotificationMessage{MessageID: uuid.NewString()}}, + {"DataFlowSuspendMessage", DataFlowSuspendMessage{MessageID: uuid.NewString(), Reason: "maintenance"}}, + {"DataFlowSuspendMessage", DataFlowSuspendMessage{MessageID: uuid.NewString()}}, + {"DataFlowResumeMessage", DataFlowResumeMessage{MessageID: uuid.NewString(), DataAddress: address}}, + {"DataFlowResumeMessage", DataFlowResumeMessage{MessageID: uuid.NewString()}}, + {"DataFlowTerminateMessage", DataFlowTerminateMessage{MessageID: uuid.NewString(), Reason: "error"}}, + {"DataFlowStatusMessage", DataFlowStatusMessage{MessageID: uuid.NewString(), DataFlowID: "flow-1", State: Started, DataAddress: address}}, + {"DataFlowStatusMessage", DataFlowStatusMessage{MessageID: uuid.NewString(), DataFlowID: "flow-1", State: Terminated, Error: "failed"}}, + {"DataFlowStatusResponseMessage", DataFlowStatusResponseMessage{DataFlowID: "flow-1", State: Preparing}}, + } + for _, tc := range tests { + t.Run(tc.schema, func(t *testing.T) { + assertConforms(t, schemas, tc.schema, tc.message) + }) + } + + t.Run("every state", func(t *testing.T) { + for state := range stateNames { + assertConforms(t, schemas, "DataFlowStatusMessage", DataFlowStatusMessage{MessageID: "m", DataFlowID: "f", State: state}) + assertConforms(t, schemas, "DataFlowStatusResponseMessage", DataFlowStatusResponseMessage{DataFlowID: "f", State: state}) + } + }) } -func Test_Message_InvalidTransferType(t *testing.T) { - msg := newBaseMessage() - msg.TransferType = TransferType{} - err := msg.Validate() - assert.ErrorIs(t, err, ErrValidation) +func Test_Messages_SchemaRejectsMissingRequired(t *testing.T) { + schemas := loadSchemas(t) + for _, name := range []string{"DataFlowPrepareMessage", "DataFlowStartMessage", "DataFlowSuspendMessage", "DataFlowStatusMessage"} { + instance, err := jsonschema.UnmarshalJSON(strings.NewReader(`{"reason": "x"}`)) + require.NoError(t, err) + assert.Error(t, schemas[name].Validate(instance), name) + } + + // the DSP DataAddress schema is applied: @type is required there, which DataAddress.MarshalJSON always adds + instance, err := jsonschema.UnmarshalJSON(strings.NewReader(`{"messageId": "m", "dataAddress": {"endpointType": "t"}}`)) + require.NoError(t, err) + assert.Error(t, schemas["DataFlowStartedNotificationMessage"].Validate(instance)) } -func Test_StartMessage_Success(t *testing.T) { - msg := newBaseMessage() - startMsg := DataFlowStartMessage{DataFlowBaseMessage: msg} - err := startMsg.Validate() - assert.NoError(t, err) +// Test_Messages_DecodeSpecExamples decodes the non-normative examples of the specification. +func Test_Messages_DecodeSpecExamples(t *testing.T) { + start := `{ + "messageId": "b1d5f9e2-3c4b-4f7a-9c3e-2f1e5d6c7b8a", + "participantId": "provider-participant-id", + "counterPartyId": "consumer-participant-id", + "dataspaceContext": "test-dataspace-context", + "dataFlowId": "test-transfer-process-id", + "agreementId": "test-agreement-id", + "datasetId": "asset-id", + "profile": "https://w3id.org/dspace-sig/profile/http-pull", + "claims": {"membership": "active", "sub": "subject"}, + "dataAddress": { + "endpointType": "https://w3id.org/idsa/v4.1/HTTP", + "endpoint": "http://dataplane.provider.com/api/public", + "endpointProperties": [ + {"name": "authorization", "value": ""}, + {"name": "authType", "value": "bearer"} + ] + }, + "labels": ["gold", "blue"], + "metadata": {"bucketName": "sourceBucket", "region": "westeurope"} + }` + var message DataFlowStartMessage + require.NoError(t, json.Unmarshal([]byte(start), &message)) + assert.NoError(t, message.Validate()) + assert.Equal(t, "test-transfer-process-id", message.DataFlowID) + assert.Equal(t, "https://w3id.org/dspace-sig/profile/http-pull", message.Profile) + assert.Equal(t, "active", message.Claims["membership"]) + assert.Equal(t, []string{"gold", "blue"}, message.Labels) + assert.Equal(t, "westeurope", message.Metadata["region"]) + token, ok := message.DataAddress.Property("authorization") + assert.True(t, ok) + assert.Equal(t, "", token) + + status := `{"messageId": "m", "dataFlowId": "dataFlow-123", "state": "PREPARED", "error": ""}` + var statusMessage DataFlowStatusMessage + require.NoError(t, json.Unmarshal([]byte(status), &statusMessage)) + assert.Equal(t, Prepared, statusMessage.State) } -func Test_StartMessage_MissingProperties(t *testing.T) { - startMsg := DataFlowStartMessage{DataFlowBaseMessage: DataFlowBaseMessage{ - MessageID: "test-id", - }} - err := startMsg.Validate() - assert.ErrorIs(t, err, ErrValidation) +func Test_DataFlowState_JSON(t *testing.T) { + data, err := json.Marshal(Initialized) + require.NoError(t, err) + assert.JSONEq(t, `"INITIALIZED"`, string(data)) + + var state DataFlowState + require.NoError(t, json.Unmarshal([]byte(`"SUSPENDED"`), &state)) + assert.Equal(t, Suspended, state) + + assert.Error(t, json.Unmarshal([]byte(`"UNINITIALIZED"`), &state)) + assert.Error(t, json.Unmarshal([]byte(`300`), &state)) + _, err = json.Marshal(DataFlowState(42)) + assert.Error(t, err) } -func Test_StartMessage_MissingSourceDataAddress(t *testing.T) { - startMsg := DataFlowStartMessage{DataFlowBaseMessage: newBaseMessage()} +func Test_Message_MissingProperties(t *testing.T) { + msg := DataFlowPrepareMessage{DataFlowBaseMessage: DataFlowBaseMessage{MessageID: "test-id"}} + assert.ErrorIs(t, msg.Validate(), ErrValidation) +} + +func Test_Message_Success(t *testing.T) { + msg := DataFlowPrepareMessage{DataFlowBaseMessage: newBaseMessage()} + assert.NoError(t, msg.Validate()) +} + +func Test_Message_RequiredFields(t *testing.T) { + tests := map[string]func(m *DataFlowBaseMessage){ + "messageId": func(m *DataFlowBaseMessage) { m.MessageID = "" }, + "participantId": func(m *DataFlowBaseMessage) { m.ParticipantID = "" }, + "counterPartyId": func(m *DataFlowBaseMessage) { m.CounterPartyID = "" }, + "dataspaceContext": func(m *DataFlowBaseMessage) { m.DataspaceContext = "" }, + "dataFlowId": func(m *DataFlowBaseMessage) { m.DataFlowID = "" }, + "agreementId": func(m *DataFlowBaseMessage) { m.AgreementID = "" }, + "datasetId": func(m *DataFlowBaseMessage) { m.DatasetID = "" }, + "profile": func(m *DataFlowBaseMessage) { m.Profile = "" }, + "claims": func(m *DataFlowBaseMessage) { m.Claims = nil }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + base := newBaseMessage() + mutate(&base) + msg := DataFlowStartMessage{DataFlowBaseMessage: base} + assert.ErrorIs(t, msg.Validate(), ErrValidation) + }) + } +} +func Test_Message_EmptyClaims(t *testing.T) { + base := newBaseMessage() + base.Claims = map[string]any{} + msg := DataFlowPrepareMessage{DataFlowBaseMessage: base} + assert.NoError(t, msg.Validate()) +} + +func Test_StartMessage_Success(t *testing.T) { + startMsg := DataFlowStartMessage{DataFlowBaseMessage: newBaseMessage(), DataAddress: newDataAddress()} assert.NoError(t, startMsg.Validate()) } -func Test_PrepareMessage_Success(t *testing.T) { - msg := newBaseMessage() - startMsg := DataFlowPrepareMessage{DataFlowBaseMessage: msg} - err := startMsg.Validate() - assert.NoError(t, err) +func Test_StartMessage_WithoutDataAddress(t *testing.T) { + startMsg := DataFlowStartMessage{DataFlowBaseMessage: newBaseMessage()} + assert.NoError(t, startMsg.Validate()) } -func Test_PrepareMessage_MissingProperties(t *testing.T) { - startMsg := DataFlowPrepareMessage{DataFlowBaseMessage: DataFlowBaseMessage{ - MessageID: "test-id", - }} - err := startMsg.Validate() - assert.ErrorIs(t, err, ErrValidation) +func Test_StartMessage_InvalidDataAddress(t *testing.T) { + startMsg := DataFlowStartMessage{DataFlowBaseMessage: newBaseMessage(), DataAddress: &DataAddress{Endpoint: "https://example.com"}} + assert.ErrorIs(t, startMsg.Validate(), ErrValidation) } -func Test_TransitionMessage_Success(t *testing.T) { - msg := DataFlowTransitionMessage{} - assert.NoError(t, msg.Validate()) +func Test_SignalMessages_RequireMessageID(t *testing.T) { + assert.ErrorIs(t, (&DataFlowStartedNotificationMessage{}).Validate(), ErrValidation) + assert.ErrorIs(t, (&DataFlowSuspendMessage{Reason: "x"}).Validate(), ErrValidation) + assert.ErrorIs(t, (&DataFlowResumeMessage{}).Validate(), ErrValidation) + assert.ErrorIs(t, (&DataFlowTerminateMessage{Reason: "x"}).Validate(), ErrValidation) - msg2 := DataFlowTransitionMessage{Reason: "test-reason"} - assert.NoError(t, msg2.Validate()) + assert.NoError(t, (&DataFlowStartedNotificationMessage{MessageID: "m"}).Validate()) + assert.NoError(t, (&DataFlowSuspendMessage{MessageID: "m"}).Validate()) + assert.NoError(t, (&DataFlowResumeMessage{MessageID: "m"}).Validate()) + assert.NoError(t, (&DataFlowTerminateMessage{MessageID: "m"}).Validate()) } func newBaseMessage() DataFlowBaseMessage { @@ -87,14 +262,21 @@ func newBaseMessage() DataFlowBaseMessage { ParticipantID: uuid.New().String(), CounterPartyID: uuid.New().String(), DataspaceContext: uuid.New().String(), - ProcessID: uuid.New().String(), + DataFlowID: uuid.New().String(), AgreementID: uuid.New().String(), DatasetID: uuid.New().String(), - CallbackAddress: CallbackURL{Scheme: "http", Host: "test.com", Path: "/callback"}, - TransferType: TransferType{ - DestinationType: "test-type", - FlowType: "pull", + Profile: pullProfile, + Claims: map[string]any{"membership": "active"}, + } +} + +func newDataAddress() *DataAddress { + return &DataAddress{ + EndpointType: "https://w3id.org/idsa/v4.1/HTTP", + Endpoint: "https://example.com/api/data", + EndpointProperties: []EndpointProperty{ + {Name: "authorization", Value: "token"}, + {Name: "authType", Value: "bearer"}, }, - DataAddress: &DataAddress{}, } } diff --git a/pkg/dsdk/model.go b/pkg/dsdk/model.go index 7723e7c..ab250bf 100644 --- a/pkg/dsdk/model.go +++ b/pkg/dsdk/model.go @@ -3,117 +3,209 @@ package dsdk import ( "encoding/json" "fmt" - "net/url" + "strings" "time" ) -type FlowType string - const ( - Pull FlowType = "pull" - Push FlowType = "push" - TypeKey = "@type" - DataAddressType = "DataAddress" - EndpointKey = "endpoint" - EndpointType = "endpointType" - EndpointProperties = "endpointProperties" + dataAddressType = "DataAddress" + endpointPropertyType = "EndpointProperty" + typeKey = "@type" ) +// DataAddress conveys information to access a wire protocol endpoint, serialized as defined by the Dataspace Protocol. type DataAddress struct { - Properties map[string]any `json:"properties"` + EndpointType string `json:"endpointType" validate:"required"` + Endpoint string `json:"endpoint,omitempty"` + EndpointProperties []EndpointProperty `json:"endpointProperties,omitempty" validate:"omitempty,min=1,dive"` } -func NewDataAddressBuilder() *DataAddressBuilder { - return &DataAddressBuilder{ - properties: make(map[string]any), +// EndpointProperty is a name/value pair of a DataAddress. +type EndpointProperty struct { + Name string `json:"name" validate:"required"` + Value string `json:"value"` +} + +// Property returns the value of the endpoint property with the given name. +func (d *DataAddress) Property(name string) (string, bool) { + if d == nil { + return "", false + } + for _, p := range d.EndpointProperties { + if p.Name == name { + return p.Value, true + } } + return "", false } -type DataAddressBuilder struct { - properties map[string]any +// Copy returns a deep copy of the data address, or nil if d is nil. +func (d *DataAddress) Copy() *DataAddress { + if d == nil { + return nil + } + c := *d + if d.EndpointProperties != nil { + c.EndpointProperties = append([]EndpointProperty(nil), d.EndpointProperties...) + } + return &c } -func (b *DataAddressBuilder) Property(key string, value any) *DataAddressBuilder { - b.properties[key] = value - return b +type dataAddressJSON struct { + Type string `json:"@type"` + EndpointType string `json:"endpointType"` + Endpoint string `json:"endpoint,omitempty"` + EndpointProperties []endpointPropertyJSON `json:"endpointProperties,omitempty"` +} + +type endpointPropertyJSON struct { + Type string `json:"@type"` + Name string `json:"name"` + Value string `json:"value"` } -func (b *DataAddressBuilder) EndpointProperty(key string, typeVal string, value any) *DataAddressBuilder { - endpointProps := b.properties[EndpointProperties] - if endpointProps == nil { - endpointProps = make([]any, 0) - b.properties[EndpointProperties] = endpointProps +// MarshalJSON adds the JSON-LD @type values required by the DSP DataAddress schema. +func (d DataAddress) MarshalJSON() ([]byte, error) { + out := dataAddressJSON{ + Type: dataAddressType, + EndpointType: d.EndpointType, + Endpoint: d.Endpoint, + } + for _, p := range d.EndpointProperties { + out.EndpointProperties = append(out.EndpointProperties, endpointPropertyJSON{Type: endpointPropertyType, Name: p.Name, Value: p.Value}) + } + return json.Marshal(out) +} + +// UnmarshalJSON accepts a DSP DataAddress; the @type values are optional on input. +func (d *DataAddress) UnmarshalJSON(data []byte) error { + var in dataAddressJSON + if err := json.Unmarshal(data, &in); err != nil { + return err + } + if in.Type != "" && in.Type != dataAddressType { + return fmt.Errorf("invalid DataAddress %s: %q", typeKey, in.Type) } - if endpoints, ok := endpointProps.([]any); ok { - props := map[string]any{ - "key": key, - "type": typeVal, - "value": value, + *d = DataAddress{EndpointType: in.EndpointType, Endpoint: in.Endpoint} + for _, p := range in.EndpointProperties { + if p.Type != "" && p.Type != endpointPropertyType { + return fmt.Errorf("invalid EndpointProperty %s: %q", typeKey, p.Type) } - endpoints = append(endpoints, props) - b.properties[EndpointProperties] = endpoints - } else { - panic("endpoint properties is not an array") + d.EndpointProperties = append(d.EndpointProperties, EndpointProperty{Name: p.Name, Value: p.Value}) } + return nil +} + +// DataAddressBuilder builds a DataAddress. +type DataAddressBuilder struct { + address DataAddress +} + +func NewDataAddressBuilder() *DataAddressBuilder { + return &DataAddressBuilder{} +} + +func (b *DataAddressBuilder) EndpointType(endpointType string) *DataAddressBuilder { + b.address.EndpointType = endpointType return b } -func (b *DataAddressBuilder) Properties(props map[string]any) *DataAddressBuilder { - for k, v := range props { - b.properties[k] = v - } +func (b *DataAddressBuilder) Endpoint(endpoint string) *DataAddressBuilder { + b.address.Endpoint = endpoint + return b +} + +// Property adds an endpoint property. +func (b *DataAddressBuilder) Property(name string, value string) *DataAddressBuilder { + b.address.EndpointProperties = append(b.address.EndpointProperties, EndpointProperty{Name: name, Value: value}) return b } func (b *DataAddressBuilder) Build() (*DataAddress, error) { - if b.properties[TypeKey] == nil { - b.properties[TypeKey] = DataAddressType + if b.address.EndpointType == "" { + return nil, NewValidationError("endpointType is required") + } + for _, p := range b.address.EndpointProperties { + if p.Name == "" { + return nil, NewValidationError("endpoint property name is required") + } } + return b.address.Copy(), nil +} + +// DataFlowState is the state of a data flow. It is stored as an int and serialized as the specification's string enum. +type DataFlowState int - return &DataAddress{ - Properties: b.properties, - }, nil +const ( + Initialized DataFlowState = 0 + Preparing DataFlowState = 50 + Prepared DataFlowState = 100 + Starting DataFlowState = 150 + Started DataFlowState = 200 + Completed DataFlowState = 250 + Suspended DataFlowState = 300 + Terminated DataFlowState = 350 +) + +var stateNames = map[DataFlowState]string{ + Initialized: "INITIALIZED", + Preparing: "PREPARING", + Prepared: "PREPARED", + Starting: "STARTING", + Started: "STARTED", + Completed: "COMPLETED", + Suspended: "SUSPENDED", + Terminated: "TERMINATED", } -type TransferType struct { - DestinationType string `json:"destinationType" validate:"required"` - FlowType FlowType `json:"flowType" validate:"required"` +func (s DataFlowState) String() string { + if name, ok := stateNames[s]; ok { + return name + } + return fmt.Sprintf("UNKNOWN(%d)", int(s)) } -type DataFlowState int +// ParseDataFlowState returns the state for the given specification name. +func ParseDataFlowState(name string) (DataFlowState, error) { + for state, n := range stateNames { + if n == name { + return state, nil + } + } + return 0, fmt.Errorf("%w: unknown data flow state %q", ErrInvalidInput, name) +} -func (s DataFlowState) String() string { - switch s { - case Uninitialized: - return "UNINITIALIZED" - case Preparing: - return "PREPARING" - case Prepared: - return "PREPARED" - case Starting: - return "STARTING" - case Started: - return "STARTED" - case Completed: - return "COMPLETED" - case Suspended: - return "SUSPENDED" - case Terminated: - return "TERMINATED" - default: - return fmt.Sprintf("UNKNOWN(%d)", int(s)) +func (s DataFlowState) MarshalJSON() ([]byte, error) { + name, ok := stateNames[s] + if !ok { + return nil, fmt.Errorf("unknown data flow state %d", int(s)) } + return json.Marshal(name) } +func (s *DataFlowState) UnmarshalJSON(data []byte) error { + var name string + if err := json.Unmarshal(data, &name); err != nil { + return err + } + state, err := ParseDataFlowState(name) + if err != nil { + return err + } + *s = state + return nil +} + +// SuspendedBy records which party suspended a data flow. Only that party may resume it. +type SuspendedBy string + const ( - Uninitialized DataFlowState = 0 - Preparing DataFlowState = 50 - Prepared DataFlowState = 100 - Starting DataFlowState = 150 - Started DataFlowState = 200 - Completed DataFlowState = 250 - Suspended DataFlowState = 300 - Terminated DataFlowState = 350 + // SuspendedByNone is set when the flow is not suspended. + SuspendedByNone SuspendedBy = "" + // SuspendedByLocal is set when the data plane suspended the flow itself. + SuspendedByLocal SuspendedBy = "local" + // SuspendedByRemote is set when the control plane suspended the flow via the signaling API. + SuspendedByRemote SuspendedBy = "remote" ) type DataFlow struct { @@ -128,26 +220,80 @@ type DataFlow struct { ParticipantID string DataspaceContext string CounterPartyID string - CallbackAddress CallbackURL - TransferType TransferType + Profile string + Claims map[string]any + Labels []string + Metadata map[string]any + ControlPlaneID string + SuspendedBy SuspendedBy State DataFlowState StateCount uint StateTimestamp int64 - SourceDataAddress DataAddress - DestinationDataAddress DataAddress + SourceDataAddress *DataAddress + DestinationDataAddress *DataAddress ErrorDetail string } -func (df *DataFlow) TransitionToPreparing() error { - if df.State == Preparing { +func (df *DataFlow) profileSuffix() string { + return df.Profile[strings.LastIndex(df.Profile, "-")+1:] +} + +// IsPush returns true if the profile denotes a push transfer, i.e. its last '-' token is "push" (case-insensitive). +func (df *DataFlow) IsPush() bool { + return strings.EqualFold(df.profileSuffix(), "push") +} + +// IsPull returns true if the profile denotes a pull transfer, i.e. its last '-' token is "pull" (case-insensitive). +func (df *DataFlow) IsPull() bool { + return strings.EqualFold(df.profileSuffix(), "pull") +} + +// Copy returns a deep copy of the data flow. Claims and metadata values are copied shallowly. +func (df *DataFlow) Copy() *DataFlow { + c := *df + c.Claims = copyMap(df.Claims) + c.Metadata = copyMap(df.Metadata) + if df.Labels != nil { + c.Labels = append([]string(nil), df.Labels...) + } + c.SourceDataAddress = df.SourceDataAddress.Copy() + c.DestinationDataAddress = df.DestinationDataAddress.Copy() + return &c +} + +func copyMap(m map[string]any) map[string]any { + if m == nil { return nil } - if df.State != Uninitialized { - return fmt.Errorf("%w: cannot transition from %v to PREPARING", ErrInvalidTransition, df.State) + c := make(map[string]any, len(m)) + for k, v := range m { + c[k] = v } - df.State = Preparing + return c +} + +func (df *DataFlow) isFinal() bool { + return df.State == Completed || df.State == Terminated +} + +func (df *DataFlow) transition(state DataFlowState) { + df.State = state df.StateTimestamp = time.Now().UnixMilli() df.StateCount++ +} + +func (df *DataFlow) invalidTransition(to DataFlowState) error { + return fmt.Errorf("%w: cannot transition from %v to %v", ErrInvalidTransition, df.State, to) +} + +func (df *DataFlow) TransitionToPreparing() error { + if df.State == Preparing { + return nil + } + if df.State != Initialized { + return df.invalidTransition(Preparing) + } + df.transition(Preparing) return nil } @@ -155,12 +301,10 @@ func (df *DataFlow) TransitionToPrepared() error { if df.State == Prepared { return nil } - if df.State != Uninitialized && df.State != Preparing { - return fmt.Errorf("%w: cannot transition from %v to PREPARED", ErrInvalidTransition, df.State) + if df.State != Initialized && df.State != Preparing { + return df.invalidTransition(Prepared) } - df.State = Prepared - df.StateTimestamp = time.Now().UnixMilli() - df.StateCount++ + df.transition(Prepared) return nil } @@ -168,40 +312,54 @@ func (df *DataFlow) TransitionToStarting() error { if df.State == Starting { return nil } - if df.State != Uninitialized && df.State != Prepared { - return fmt.Errorf("%w: cannot transition from %v to STARTING", ErrInvalidTransition, df.State) + if df.State != Initialized && df.State != Prepared { + return df.invalidTransition(Starting) } - df.State = Starting - df.StateTimestamp = time.Now().UnixMilli() - df.StateCount++ + df.transition(Starting) return nil } +// TransitionToStarted transitions a new, prepared or starting flow to STARTED. Use TransitionToResumed for a +// suspended flow. func (df *DataFlow) TransitionToStarted() error { if df.State == Started { return nil } - if df.State != Uninitialized && df.State != Prepared && df.State != Starting && df.State != Suspended { - return fmt.Errorf("%w: cannot transition from %v to STARTED", ErrInvalidTransition, df.State) + if df.State != Initialized && df.State != Prepared && df.State != Starting { + return df.invalidTransition(Started) } - df.State = Started - df.StateTimestamp = time.Now().UnixMilli() - df.StateCount++ + df.transition(Started) return nil } -func (df *DataFlow) TransitionToSuspended(reason string) error { +// TransitionToSuspended transitions a STARTED flow to SUSPENDED and records who suspended it. +func (df *DataFlow) TransitionToSuspended(reason string, by SuspendedBy) error { if df.State == Suspended { return nil } if df.State != Started { - return fmt.Errorf("%w: cannot transition from %v to SUSPENDED", ErrInvalidTransition, df.State) + return df.invalidTransition(Suspended) + } + if by != SuspendedByLocal && by != SuspendedByRemote { + return fmt.Errorf("%w: invalid suspending party %q", ErrInvalidInput, by) } - df.State = Suspended - //todo: what to do with the reason string? + df.SuspendedBy = by df.ErrorDetail = reason - df.StateTimestamp = time.Now().UnixMilli() - df.StateCount++ + df.transition(Suspended) + return nil +} + +// TransitionToResumed transitions a SUSPENDED flow back to STARTED. Only the party that suspended the flow may resume it. +func (df *DataFlow) TransitionToResumed(by SuspendedBy) error { + if df.State != Suspended { + return df.invalidTransition(Started) + } + if df.SuspendedBy != by { + return fmt.Errorf("%w: data flow %s was suspended by the %s party and can only be resumed by it", ErrInvalidTransition, df.ID, df.SuspendedBy) + } + df.SuspendedBy = SuspendedByNone + df.ErrorDetail = "" + df.transition(Started) return nil } @@ -210,23 +368,22 @@ func (df *DataFlow) TransitionToCompleted() error { return nil } if df.State != Started { - return fmt.Errorf("%w: cannot transition from %v to COMPLETED", ErrInvalidTransition, df.State) + return df.invalidTransition(Completed) } - df.State = Completed - df.StateTimestamp = time.Now().UnixMilli() - df.StateCount++ + df.transition(Completed) return nil } +// TransitionToTerminated transitions any non-final flow to TERMINATED. COMPLETED is final and cannot be terminated. func (df *DataFlow) TransitionToTerminated(reason string) error { if df.State == Terminated { - return nil // todo: does returning an error make sense here? + return nil + } + if df.State == Completed { + return df.invalidTransition(Terminated) } - // Any state can transition to terminated - df.State = Terminated df.ErrorDetail = reason - df.StateTimestamp = time.Now().UnixMilli() - df.StateCount++ + df.transition(Terminated) return nil } @@ -273,6 +430,36 @@ func (b *DataFlowBuilder) CounterpartyID(id string) *DataFlowBuilder { return b } +func (b *DataFlowBuilder) Profile(profile string) *DataFlowBuilder { + b.dataFlow.Profile = profile + return b +} + +func (b *DataFlowBuilder) Claims(claims map[string]any) *DataFlowBuilder { + b.dataFlow.Claims = claims + return b +} + +func (b *DataFlowBuilder) Labels(labels []string) *DataFlowBuilder { + b.dataFlow.Labels = labels + return b +} + +func (b *DataFlowBuilder) Metadata(metadata map[string]any) *DataFlowBuilder { + b.dataFlow.Metadata = metadata + return b +} + +func (b *DataFlowBuilder) ControlPlaneID(id string) *DataFlowBuilder { + b.dataFlow.ControlPlaneID = id + return b +} + +func (b *DataFlowBuilder) SuspendedBy(by SuspendedBy) *DataFlowBuilder { + b.dataFlow.SuspendedBy = by + return b +} + func (b *DataFlowBuilder) State(state DataFlowState) *DataFlowBuilder { b.dataFlow.State = state return b @@ -288,26 +475,16 @@ func (b *DataFlowBuilder) StateTimestamp(timestamp int64) *DataFlowBuilder { return b } -func (b *DataFlowBuilder) SourceDataAddress(address DataAddress) *DataFlowBuilder { +func (b *DataFlowBuilder) SourceDataAddress(address *DataAddress) *DataFlowBuilder { b.dataFlow.SourceDataAddress = address return b } -func (b *DataFlowBuilder) DestinationDataAddress(address DataAddress) *DataFlowBuilder { +func (b *DataFlowBuilder) DestinationDataAddress(address *DataAddress) *DataFlowBuilder { b.dataFlow.DestinationDataAddress = address return b } -func (b *DataFlowBuilder) CallbackAddress(callback CallbackURL) *DataFlowBuilder { - b.dataFlow.CallbackAddress = callback - return b -} - -func (b *DataFlowBuilder) TransferType(transferType TransferType) *DataFlowBuilder { - b.dataFlow.TransferType = transferType - return b -} - func (b *DataFlowBuilder) ErrorDetail(error string) *DataFlowBuilder { b.dataFlow.ErrorDetail = error return b @@ -359,16 +536,8 @@ func (b *DataFlowBuilder) Build() (*DataFlow, error) { validationErrs = append(validationErrs, "CounterPartyID is required") } - if b.dataFlow.CallbackAddress.URL().String() == "" { - validationErrs = append(validationErrs, "CallbackAddress is required") - } - - if b.dataFlow.TransferType.DestinationType == "" { - validationErrs = append(validationErrs, "TransferType destination type is required") - } - - if b.dataFlow.TransferType.FlowType == "" { - validationErrs = append(validationErrs, "TransferType flow type is required") + if b.dataFlow.Profile == "" { + validationErrs = append(validationErrs, "Profile is required") } if len(validationErrs) > 0 { @@ -377,39 +546,3 @@ func (b *DataFlowBuilder) Build() (*DataFlow, error) { return &b.dataFlow, nil } - -type CallbackURL url.URL - -func (u CallbackURL) MarshalJSON() ([]byte, error) { - // Convert to standard url.URL and get the string representation - stdURL := url.URL(u) - return json.Marshal(stdURL.String()) -} - -func (u *CallbackURL) UnmarshalJSON(data []byte) error { - var urlStr string - if err := json.Unmarshal(data, &urlStr); err != nil { - return err - } - - parsedURL, err := url.Parse(urlStr) - if err != nil { - return err - } - - *u = CallbackURL(*parsedURL) - return nil -} - -func (u *CallbackURL) URL() *url.URL { - if u == nil { - return nil - } - - urlCopy := url.URL(*u) - return &urlCopy -} - -func (u CallbackURL) IsEmpty() bool { - return u.Scheme == "" && u.Host == "" -} diff --git a/pkg/dsdk/model_test.go b/pkg/dsdk/model_test.go index 8eefbe1..737b623 100644 --- a/pkg/dsdk/model_test.go +++ b/pkg/dsdk/model_test.go @@ -2,7 +2,6 @@ package dsdk import ( "encoding/json" - "net/url" "testing" "time" @@ -12,83 +11,154 @@ import ( ) func Test_dataFlowStartSerialize(t *testing.T) { - callbackURL, _ := url.Parse("https://example.com/callback") - build, _ := NewDataAddressBuilder().Property("foo", "bar").Build() + address, err := NewDataAddressBuilder().EndpointType("https://w3id.org/idsa/v4.1/HTTP").Property("foo", "bar").Build() + require.NoError(t, err) original := DataFlowStartMessage{ DataFlowBaseMessage: DataFlowBaseMessage{ MessageID: uuid.New().String(), ParticipantID: "participant123", CounterPartyID: uuid.New().String(), DataspaceContext: uuid.New().String(), - ProcessID: uuid.New().String(), + DataFlowID: uuid.New().String(), AgreementID: "agreement456", DatasetID: uuid.New().String(), - CallbackAddress: CallbackURL(*callbackURL), - TransferType: TransferType{ - DestinationType: "PULL", - FlowType: FlowType("PULL"), - }, - DataAddress: build, + Profile: pushProfile, + Claims: map[string]any{"sub": "subject"}, + Labels: []string{"gold"}, + Metadata: map[string]any{"bucketName": "destinationBucket"}, }, + DataAddress: address, } jsonData, err := json.Marshal(original) - assert.NoError(t, err) + require.NoError(t, err) + + var raw map[string]any + require.NoError(t, json.Unmarshal(jsonData, &raw)) + for _, key := range []string{"messageId", "participantId", "counterPartyId", "dataspaceContext", "dataFlowId", "agreementId", "datasetId", "profile", "claims", "labels", "metadata", "dataAddress"} { + assert.Contains(t, raw, key) + } var decoded DataFlowStartMessage - err = json.Unmarshal(jsonData, &decoded) - assert.NoError(t, err) - - assert.Equal(t, original.ParticipantID, decoded.ParticipantID, "ParticipantID should be equal") - assert.Equal(t, original.AgreementID, decoded.AgreementID, "AgreementID should be equal") - assert.Equal(t, original.CallbackAddress, decoded.CallbackAddress, "CallbackAddress should be equal") - assert.Equal(t, original.TransferType, decoded.TransferType, "TransferType should be equal") - assert.Equal(t, original.DataAddress, decoded.DataAddress, "DataAddress should be equal") - assert.Equal(t, original, original) + require.NoError(t, json.Unmarshal(jsonData, &decoded)) + assert.Equal(t, original, decoded) } -func TestDataFlowBuilder_Build(t *testing.T) { - validURL, err := url.Parse("http://example.com/callback") - if err != nil { - t.Fatal("failed to parse test URL:", err) +func Test_DataAddress_JSON(t *testing.T) { + address := newDataAddress() + data, err := json.Marshal(address) + require.NoError(t, err) + assert.JSONEq(t, `{ + "@type": "DataAddress", + "endpointType": "https://w3id.org/idsa/v4.1/HTTP", + "endpoint": "https://example.com/api/data", + "endpointProperties": [ + {"@type": "EndpointProperty", "name": "authorization", "value": "token"}, + {"@type": "EndpointProperty", "name": "authType", "value": "bearer"} + ] + }`, string(data)) + + var decoded DataAddress + require.NoError(t, json.Unmarshal(data, &decoded)) + assert.Equal(t, *address, decoded) + + // @type is optional on input + require.NoError(t, json.Unmarshal([]byte(`{"endpointType": "t", "endpointProperties": [{"name": "a", "value": "b"}]}`), &decoded)) + assert.Equal(t, DataAddress{EndpointType: "t", EndpointProperties: []EndpointProperty{{Name: "a", Value: "b"}}}, decoded) + + assert.Error(t, json.Unmarshal([]byte(`{"@type": "Other", "endpointType": "t"}`), &decoded)) +} + +func TestDataAddressBuilder(t *testing.T) { + address, err := NewDataAddressBuilder(). + EndpointType("https://w3id.org/idsa/v4.1/HTTP"). + Endpoint("https://example.com"). + Property("authorization", "token"). + Property("authType", "bearer"). + Build() + require.NoError(t, err) + assert.Equal(t, "https://w3id.org/idsa/v4.1/HTTP", address.EndpointType) + assert.Equal(t, "https://example.com", address.Endpoint) + assert.Equal(t, []EndpointProperty{{Name: "authorization", Value: "token"}, {Name: "authType", Value: "bearer"}}, address.EndpointProperties) + + value, ok := address.Property("authType") + assert.True(t, ok) + assert.Equal(t, "bearer", value) + _, ok = address.Property("missing") + assert.False(t, ok) + + _, err = NewDataAddressBuilder().Endpoint("https://example.com").Build() + assert.ErrorIs(t, err, ErrValidation) + + _, err = NewDataAddressBuilder().EndpointType("t").Property("", "value").Build() + assert.ErrorIs(t, err, ErrValidation) +} + +func TestDataFlow_PushPull(t *testing.T) { + tests := []struct { + profile string + push, pull bool + }{ + {"https://w3id.org/dspace-sig/profile/s3-push", true, false}, + {"https://w3id.org/dspace-sig/profile/http-pull", false, true}, + {"HTTP-PUSH", true, false}, + {"FileSystem-PULL", false, true}, + {"Push", true, false}, + {"http-pull-something", false, false}, + {"", false, false}, } + for _, tc := range tests { + t.Run(tc.profile, func(t *testing.T) { + flow := DataFlow{Profile: tc.profile} + assert.Equal(t, tc.push, flow.IsPush()) + assert.Equal(t, tc.pull, flow.IsPull()) + }) + } +} +func TestDataFlow_Copy(t *testing.T) { + flow := &DataFlow{ + ID: "id", + Claims: map[string]any{"a": "b"}, + Labels: []string{"gold"}, + Metadata: map[string]any{"c": "d"}, + SourceDataAddress: newDataAddress(), + } + c := flow.Copy() + c.Claims["a"] = "changed" + c.Labels[0] = "changed" + c.Metadata["c"] = "changed" + c.SourceDataAddress.EndpointProperties[0].Value = "changed" + + assert.Equal(t, "b", flow.Claims["a"]) + assert.Equal(t, "gold", flow.Labels[0]) + assert.Equal(t, "d", flow.Metadata["c"]) + assert.Equal(t, "token", flow.SourceDataAddress.EndpointProperties[0].Value) + assert.Nil(t, c.DestinationDataAddress) +} + +func TestDataFlowBuilder_Build(t *testing.T) { tests := map[string]struct { builder func() *DataFlowBuilder wantErr bool }{ "success with all fields": { - builder: func() *DataFlowBuilder { - return NewDataFlowBuilder(). - ID("test-id"). - UpdatedAt(int64(time.Now().Unix())). - CreatedAt(int64(time.Now().Unix())). - ParticipantID("part-123"). - DataspaceContext("ctx-123"). - CounterpartyID("counter-123"). - State(Started). - StateTimestamp(time.Now().Unix()). - SourceDataAddress(DataAddress{Properties: map[string]any{"source": "test"}}). - DestinationDataAddress(DataAddress{Properties: map[string]any{"dest": "test"}}). - CallbackAddress(CallbackURL(*validURL)). - TransferType(TransferType{ - DestinationType: "test-dest", - FlowType: Pull, - }). - RuntimeID("runtime-123") - }, + builder: createValidBuilder, wantErr: false, }, "fails with empty builder": { + builder: NewDataFlowBuilder, + wantErr: true, + }, + "fails without ID": { builder: func() *DataFlowBuilder { - return NewDataFlowBuilder() + return createValidBuilder().ID("") }, wantErr: true, }, - "fails without ID": { + "fails without profile": { builder: func() *DataFlowBuilder { - b := createValidBuilder(validURL) - return b.ID("") + return createValidBuilder().Profile("") }, wantErr: true, }, @@ -99,117 +169,43 @@ func TestDataFlowBuilder_Build(t *testing.T) { flow, err := tc.builder().Build() if tc.wantErr { - if err == nil { - t.Error("expected error, got nil") - } + assert.ErrorIs(t, err, ErrValidation) return } - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // Verify non-zero/non-empty values for required fields - if flow.ID == "" { - t.Error("ID is empty") - } - if flow.UpdatedAt == 0 { - t.Error("UpdatedAt is zero") - } - if flow.CreatedAt == 0 { - t.Error("CreatedAt is zero") - } - if flow.ParticipantID == "" { - t.Error("ParticipantID is empty") - } - if flow.DataspaceContext == "" { - t.Error("DataspaceContext is empty") - } - if flow.CounterPartyID == "" { - t.Error("CounterPartyID is empty") - } - if flow.StateTimestamp == 0 { - t.Error("StateTimestamp is zero") - } - if flow.CallbackAddress.URL() == nil { - t.Error("CallbackAddress is nil") - } - if flow.TransferType.DestinationType == "" { - t.Error("TransferType destination type is empty") - } - if flow.TransferType.FlowType == "" { - t.Error("TransferType flow type is empty") - } - }) - } -} - -func TestDataAddressBuilder_EndpointProperty(t *testing.T) { - tests := []struct { - name string - key string - typeVal string - value any - want any - }{ - { - name: "sets valid string property", - key: "endpoint", - typeVal: "string", - value: "https://api.example.com/v1/data", - want: []interface{}{map[string]interface{}{"key": "endpoint", "type": "string", "value": "https://api.example.com/v1/data"}}, - }, - { - name: "sets integer property", - key: "port", - typeVal: "int", - value: 8080, - want: []interface{}{map[string]interface{}{"key": "port", "type": "int", "value": 8080}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - builder := NewDataAddressBuilder() - result := builder.EndpointProperty(tt.key, tt.typeVal, tt.value) - - if result != builder { - t.Error("EndpointProperty should return the builder for method chaining") - } - - // Build and verify the property was set - dataAddress, err := builder.Build() - if err != nil { - t.Fatalf("Build() failed: %v", err) - } - - got, exists := dataAddress.Properties[EndpointProperties] - require.True(t, exists, "EndpointProperties not found in built DataAddress") - - // Compare the entire slice using testify - assert.Equal(t, tt.want, got, "EndpointProperty slice should match expected") - + require.NoError(t, err) + + assert.NotEmpty(t, flow.ID) + assert.NotZero(t, flow.UpdatedAt) + assert.NotZero(t, flow.CreatedAt) + assert.NotEmpty(t, flow.ParticipantID) + assert.NotEmpty(t, flow.DataspaceContext) + assert.NotEmpty(t, flow.CounterPartyID) + assert.NotZero(t, flow.StateTimestamp) + assert.Equal(t, pullProfile, flow.Profile) + assert.Equal(t, map[string]any{"sub": "subject"}, flow.Claims) + assert.Equal(t, []string{"gold"}, flow.Labels) + assert.Equal(t, map[string]any{"region": "westeurope"}, flow.Metadata) + assert.Equal(t, "cp-1", flow.ControlPlaneID) + assert.Equal(t, SuspendedByNone, flow.SuspendedBy) }) } } -// Helper function to create a valid builder for testing -func createValidBuilder(validURL *url.URL) *DataFlowBuilder { +func createValidBuilder() *DataFlowBuilder { return NewDataFlowBuilder(). ID("test-id"). - UpdatedAt(int64(time.Now().Unix())). - CreatedAt(int64(time.Now().Unix())). + UpdatedAt(time.Now().Unix()). + CreatedAt(time.Now().Unix()). ParticipantID("part-123"). DataspaceContext("ctx-123"). CounterpartyID("counter-123"). + Profile(pullProfile). + Claims(map[string]any{"sub": "subject"}). + Labels([]string{"gold"}). + Metadata(map[string]any{"region": "westeurope"}). + ControlPlaneID("cp-1"). State(Started). StateTimestamp(time.Now().Unix()). - SourceDataAddress(DataAddress{Properties: map[string]any{"source": "test"}}). - DestinationDataAddress(DataAddress{Properties: map[string]any{"dest": "test"}}). - CallbackAddress(CallbackURL(*validURL)). - TransferType(TransferType{ - DestinationType: "test-dest", - FlowType: Pull, - }). + SourceDataAddress(newDataAddress()). RuntimeID("runtime-123") } diff --git a/pkg/dsdk/model_transition_test.go b/pkg/dsdk/model_transition_test.go index 71c5b0c..2f1786e 100644 --- a/pkg/dsdk/model_transition_test.go +++ b/pkg/dsdk/model_transition_test.go @@ -15,6 +15,8 @@ package dsdk import ( "testing" "time" + + "github.com/stretchr/testify/assert" ) func TestDataFlow_transitionToPreparing(t *testing.T) { @@ -23,8 +25,8 @@ func TestDataFlow_transitionToPreparing(t *testing.T) { expectErr bool expectedErr string }{ - "valid transition from Uninitialized": { - initialState: Uninitialized, + "valid transition from Initialized": { + initialState: Initialized, expectErr: false, }, "idempotent transition from Preparing": { @@ -126,8 +128,8 @@ func TestDataFlow_transitionToPrepared(t *testing.T) { expectErr bool expectedErr string }{ - "valid transition from Uninitialized": { - initialState: Uninitialized, + "valid transition from Initialized": { + initialState: Initialized, expectErr: false, }, "valid transition from Preparing": { @@ -228,8 +230,8 @@ func TestDataFlow_transitionToStarting(t *testing.T) { expectErr bool expectedErr string }{ - "valid transition from Uninitialized": { - initialState: Uninitialized, + "valid transition from Initialized": { + initialState: Initialized, expectErr: false, }, "invalid transition from Preparing": { @@ -330,8 +332,8 @@ func TestDataFlow_transitionToStarted(t *testing.T) { expectErr bool expectedErr string }{ - "valid transition from Uninitialized": { - initialState: Uninitialized, + "valid transition from Initialized": { + initialState: Initialized, expectErr: false, }, "invalid transition from Prepared": { @@ -356,9 +358,10 @@ func TestDataFlow_transitionToStarted(t *testing.T) { expectErr: true, expectedErr: "invalid transition: cannot transition from COMPLETED to STARTED", }, - "valid transition from Suspended": { + "invalid transition from Suspended (use TransitionToResumed)": { initialState: Suspended, - expectErr: false, + expectErr: true, + expectedErr: "invalid transition: cannot transition from SUSPENDED to STARTED", }, "invalid transition from Terminated": { initialState: Terminated, @@ -430,10 +433,10 @@ func TestDataFlow_transitionToSuspended(t *testing.T) { expectErr bool expectedErr string }{ - "invalid transition from Uninitialized": { - initialState: Uninitialized, + "invalid transition from Initialized": { + initialState: Initialized, expectErr: true, - expectedErr: "invalid transition: cannot transition from UNINITIALIZED to SUSPENDED", + expectedErr: "invalid transition: cannot transition from INITIALIZED to SUSPENDED", }, "invalid transition from Preparing": { initialState: Preparing, @@ -479,7 +482,7 @@ func TestDataFlow_transitionToSuspended(t *testing.T) { initialStateCount := df.StateCount initialTimestamp := df.StateTimestamp - err := df.TransitionToSuspended("test-reason") + err := df.TransitionToSuspended("test-reason", SuspendedByRemote) if tc.expectErr { if err == nil { @@ -533,10 +536,10 @@ func TestDataFlow_transitionToCompleted(t *testing.T) { expectErr bool expectedErr string }{ - "invalid transition from Uninitialized": { - initialState: Uninitialized, + "invalid transition from Initialized": { + initialState: Initialized, expectErr: true, - expectedErr: "invalid transition: cannot transition from UNINITIALIZED to COMPLETED", + expectedErr: "invalid transition: cannot transition from INITIALIZED to COMPLETED", }, "invalid transition from Preparing": { initialState: Preparing, @@ -635,8 +638,8 @@ func TestDataFlow_transitionToTerminated(t *testing.T) { initialState DataFlowState expectErr bool }{ - "valid transition from Uninitialized": { - initialState: Uninitialized, + "valid transition from Initialized": { + initialState: Initialized, expectErr: false, }, "valid transition from Preparing": { @@ -655,9 +658,9 @@ func TestDataFlow_transitionToTerminated(t *testing.T) { initialState: Started, expectErr: false, }, - "valid transition from Completed": { + "invalid transition from Completed (final)": { initialState: Completed, - expectErr: false, + expectErr: true, }, "valid transition from Suspended": { initialState: Suspended, @@ -726,7 +729,7 @@ func TestDataFlow_transitionToTerminated(t *testing.T) { // Test to ensure timestamps are properly set and state counts increment correctly func TestDataFlow_TransitionsTimestampAndCounter(t *testing.T) { df := &DataFlow{ - State: Uninitialized, + State: Initialized, StateCount: 0, } @@ -766,7 +769,7 @@ func TestDataFlow_TransitionsTimestampAndCounter(t *testing.T) { // Test case with zero initial timestamp func TestDataFlow_TransitionFromZeroTimestamp(t *testing.T) { df := &DataFlow{ - State: Uninitialized, + State: Initialized, StateCount: 0, StateTimestamp: 0, } @@ -786,3 +789,29 @@ func TestDataFlow_TransitionFromZeroTimestamp(t *testing.T) { t.Errorf("expected state count 1, got %d", df.StateCount) } } + +func TestDataFlow_SuspendAndResume(t *testing.T) { + df := &DataFlow{ID: "flow", State: Started} + + err := df.TransitionToSuspended("maintenance", SuspendedByNone) + assert.ErrorIs(t, err, ErrInvalidInput) + assert.Equal(t, Started, df.State) + + assert.NoError(t, df.TransitionToSuspended("maintenance", SuspendedByLocal)) + assert.Equal(t, Suspended, df.State) + assert.Equal(t, SuspendedByLocal, df.SuspendedBy) + assert.Equal(t, "maintenance", df.ErrorDetail) + + // only the suspending party may resume + err = df.TransitionToResumed(SuspendedByRemote) + assert.ErrorIs(t, err, ErrInvalidTransition) + assert.Equal(t, Suspended, df.State) + + assert.NoError(t, df.TransitionToResumed(SuspendedByLocal)) + assert.Equal(t, Started, df.State) + assert.Equal(t, SuspendedByNone, df.SuspendedBy) + assert.Empty(t, df.ErrorDetail) + + // resuming a flow that is not suspended fails + assert.ErrorIs(t, df.TransitionToResumed(SuspendedByLocal), ErrInvalidTransition) +} diff --git a/pkg/dsdk/router.go b/pkg/dsdk/router.go new file mode 100644 index 0000000..22bfaaa --- /dev/null +++ b/pkg/dsdk/router.go @@ -0,0 +1,55 @@ +package dsdk + +import ( + "net/http" + "strings" +) + +// DefaultBasePath is the path prefix under which NewHandler mounts the data plane endpoints. +const DefaultBasePath = "/v1" + +type handlerConfig struct { + basePath string +} + +// HandlerOption configures the handler returned by NewHandler. +type HandlerOption func(*handlerConfig) + +// WithBasePath sets the path prefix of the data plane endpoints, e.g. "/api/signaling/v1". Use "" to mount them at +// the root. The prefix is also used for the Location header of 202 responses. +func WithBasePath(basePath string) HandlerOption { + return func(c *handlerConfig) { + c.basePath = strings.TrimSuffix(basePath, "/") + } +} + +// NewHandler returns an http.Handler that serves all data plane endpoints of the Data Plane Signaling API under the +// configured base path (DefaultBasePath unless WithBasePath is given): +// +// POST /dataflows/prepare +// POST /dataflows/start +// POST /dataflows/{id}/started +// POST /dataflows/{id}/suspend +// POST /dataflows/{id}/resume +// POST /dataflows/{id}/terminate +// POST /dataflows/{id}/completed +// GET /dataflows/{id}/status +func NewHandler(sdk *DataPlaneSDK, options ...HandlerOption) http.Handler { + config := handlerConfig{basePath: DefaultBasePath} + for _, opt := range options { + opt(&config) + } + api := &DataPlaneApi{sdk: sdk, basePath: config.basePath} + base := config.basePath + "/dataflows" + + mux := http.NewServeMux() + mux.HandleFunc("POST "+base+"/prepare", api.Prepare) + mux.HandleFunc("POST "+base+"/start", api.Start) + mux.HandleFunc("POST "+base+"/{id}/started", api.Started) + mux.HandleFunc("POST "+base+"/{id}/suspend", api.Suspend) + mux.HandleFunc("POST "+base+"/{id}/resume", api.Resume) + mux.HandleFunc("POST "+base+"/{id}/terminate", api.Terminate) + mux.HandleFunc("POST "+base+"/{id}/completed", api.Completed) + mux.HandleFunc("GET "+base+"/{id}/status", api.Status) + return mux +} diff --git a/pkg/dsdk/router_test.go b/pkg/dsdk/router_test.go new file mode 100644 index 0000000..9415100 --- /dev/null +++ b/pkg/dsdk/router_test.go @@ -0,0 +1,136 @@ +package dsdk + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func serve(t *testing.T, handler http.Handler, method, path string, body any) *httptest.ResponseRecorder { + t.Helper() + var reader *strings.Reader + switch b := body.(type) { + case nil: + reader = strings.NewReader("") + case string: + reader = strings.NewReader(b) + default: + data, err := json.Marshal(b) + require.NoError(t, err) + reader = strings.NewReader(string(data)) + } + req := httptest.NewRequest(method, path, reader) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + return rr +} + +func Test_Router_AllRoutes(t *testing.T) { + sdk, store := newTestSDK(t) + handler := NewHandler(sdk) + + // provider pull: start, suspend, resume, complete, status + start := createStartMessage(pullProfile, nil) + rr := serve(t, handler, http.MethodPost, "/v1/dataflows/start", start) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + id := start.DataFlowID + + assert.Equal(t, http.StatusOK, serve(t, handler, http.MethodPost, "/v1/dataflows/"+id+"/suspend", DataFlowSuspendMessage{MessageID: "m"}).Code) + rr = serve(t, handler, http.MethodPost, "/v1/dataflows/"+id+"/resume", DataFlowResumeMessage{MessageID: "m"}) + assert.Equal(t, http.StatusOK, rr.Code) + var status DataFlowStatusMessage + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &status)) + assert.Equal(t, Started, status.State) + + assert.Equal(t, http.StatusOK, serve(t, handler, http.MethodPost, "/v1/dataflows/"+id+"/completed", nil).Code) + rr = serve(t, handler, http.MethodGet, "/v1/dataflows/"+id+"/status", nil) + assert.Equal(t, http.StatusOK, rr.Code) + assert.JSONEq(t, `{"dataFlowId": "`+id+`", "state": "COMPLETED"}`, rr.Body.String()) + assert.Equal(t, Completed, store.get(t, id).State) + + // terminating a completed flow is rejected + assert.Equal(t, http.StatusBadRequest, serve(t, handler, http.MethodPost, "/v1/dataflows/"+id+"/terminate", DataFlowTerminateMessage{MessageID: "m"}).Code) + + // consumer push: prepare, started, terminate + prepare := createPrepareMessage(pushProfile) + assert.Equal(t, http.StatusOK, serve(t, handler, http.MethodPost, "/v1/dataflows/prepare", prepare).Code) + rr = serve(t, handler, http.MethodPost, "/v1/dataflows/"+prepare.DataFlowID+"/started", DataFlowStartedNotificationMessage{MessageID: "m"}) + assert.Equal(t, http.StatusOK, rr.Code) + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &status)) + assert.Equal(t, Started, status.State) + assert.Equal(t, prepare.DataFlowID, status.DataFlowID) + assert.Equal(t, http.StatusOK, serve(t, handler, http.MethodPost, "/v1/dataflows/"+prepare.DataFlowID+"/terminate", DataFlowTerminateMessage{MessageID: "m"}).Code) + assert.Equal(t, Terminated, store.get(t, prepare.DataFlowID).State) +} + +func Test_Router_AcceptedSetsLocation(t *testing.T) { + sdk, _ := newTestSDK(t, WithPrepareProcessor(stateProcessor(Preparing)), WithStartProcessor(stateProcessor(Starting))) + + for _, basePath := range []string{"", "/v1", "/api/signaling/v1"} { + t.Run(basePath, func(t *testing.T) { + var handler http.Handler + if basePath == "/v1" { + handler = NewHandler(sdk) + } else { + handler = NewHandler(sdk, WithBasePath(basePath+"/")) + } + + prepare := createPrepareMessage(pullProfile) + rr := serve(t, handler, http.MethodPost, basePath+"/dataflows/prepare", prepare) + assert.Equal(t, http.StatusAccepted, rr.Code) + assert.Equal(t, basePath+"/dataflows/"+prepare.DataFlowID+"/status", rr.Header().Get("Location")) + + start := createStartMessage(pullProfile, nil) + rr = serve(t, handler, http.MethodPost, basePath+"/dataflows/start", start) + assert.Equal(t, http.StatusAccepted, rr.Code) + assert.Equal(t, basePath+"/dataflows/"+start.DataFlowID+"/status", rr.Header().Get("Location")) + + assert.Equal(t, http.StatusOK, serve(t, handler, http.MethodGet, rr.Header().Get("Location"), nil).Code) + }) + } +} + +func Test_Router_Errors(t *testing.T) { + sdk, store := newTestSDK(t) + handler := NewHandler(sdk) + store.put(&DataFlow{ID: "flow", Profile: pullProfile, State: Started}) + + tests := map[string]struct { + method, path string + body any + code int + }{ + "wrong method": {http.MethodGet, "/v1/dataflows/start", nil, http.StatusMethodNotAllowed}, + "unknown path": {http.MethodPost, "/dataflows/start", "{}", http.StatusNotFound}, + "malformed json": {http.MethodPost, "/v1/dataflows/start", "{", http.StatusBadRequest}, + "suspend without body": {http.MethodPost, "/v1/dataflows/flow/suspend", nil, http.StatusBadRequest}, + "suspend without messageId": {http.MethodPost, "/v1/dataflows/flow/suspend", `{"reason": "x"}`, http.StatusBadRequest}, + "resume without body": {http.MethodPost, "/v1/dataflows/flow/resume", nil, http.StatusBadRequest}, + "terminate without body": {http.MethodPost, "/v1/dataflows/flow/terminate", nil, http.StatusBadRequest}, + "started without messageId": {http.MethodPost, "/v1/dataflows/flow/started", `{}`, http.StatusBadRequest}, + "status not found": {http.MethodGet, "/v1/dataflows/unknown/status", nil, http.StatusNotFound}, + "completed not found": {http.MethodPost, "/v1/dataflows/unknown/completed", nil, http.StatusNotFound}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + rr := serve(t, handler, tc.method, tc.path, tc.body) + assert.Equal(t, tc.code, rr.Code, rr.Body.String()) + if tc.code == http.StatusBadRequest || (tc.code == http.StatusNotFound && strings.HasPrefix(tc.path, "/v1")) { + var errorResponse ErrorResponse + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &errorResponse)) + assert.NotEmpty(t, errorResponse.Error) + } + }) + } + assert.Equal(t, Started, store.get(t, "flow").State) + + // a duplicate start is a conflict + start := createStartMessage(pullProfile, nil) + assert.Equal(t, http.StatusOK, serve(t, handler, http.MethodPost, "/v1/dataflows/start", start).Code) + assert.Equal(t, http.StatusConflict, serve(t, handler, http.MethodPost, "/v1/dataflows/start", start).Code) +} diff --git a/pkg/dsdk/testdata/schemas/AgreementResponse.schema.json b/pkg/dsdk/testdata/schemas/AgreementResponse.schema.json new file mode 100644 index 0000000..99d18f1 --- /dev/null +++ b/pkg/dsdk/testdata/schemas/AgreementResponse.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AgreementResponseSchema", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/RootAgreement" + } + ], + "$id": "AgreementResponse.schema.json", + "definitions": { + "RootAgreement": { + "type": "object", + "allOf": [ + { + "$ref": "https://w3id.org/dspace/2025/1/negotiation/contract-schema.json#/definitions/Agreement" + } + ], + "properties": { + "@context": { + "$ref": "https://w3id.org/dspace/2025/1/common/context-schema.json" + } + }, + "required": [ + "@context" + ] + } + } +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/AuthorizationEntry.schema.json b/pkg/dsdk/testdata/schemas/AuthorizationEntry.schema.json new file mode 100644 index 0000000..6f8594b --- /dev/null +++ b/pkg/dsdk/testdata/schemas/AuthorizationEntry.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "AuthorizationEntry", + "properties": { + "type": { + "type": "string", + "description": "The authorization profile", + "examples": ["oauth2_client_credentials"] + }, + "tokenEndpoint": { + "type": "string", + "description": "The token endpoint to be used for obtaining access tokens", + "examples": ["https://auth.example.com/oauth2/token"] + }, + "clientId": { + "type": "string", + "description": "The client ID for the authorization profile", + "examples": ["123456789"] + }, + "clientSecret": { + "type": "string", + "description": "The client secret for the authorization profile", + "examples": ["s3cr3t"] + } + } +} diff --git a/pkg/dsdk/testdata/schemas/ControlPlaneRegistrationMessage.schema.json b/pkg/dsdk/testdata/schemas/ControlPlaneRegistrationMessage.schema.json new file mode 100644 index 0000000..40f1787 --- /dev/null +++ b/pkg/dsdk/testdata/schemas/ControlPlaneRegistrationMessage.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "ControlPlaneRegistrationMessage", + "required": ["controlplaneId", "endpoint", "authorization"], + "properties": { + "controlplaneId": { + "type": "string", + "description": "The unique identifier of the control plane", + "examples": ["controlplane-64345"] + }, + "endpoint": { + "type": "string", + "description": "The base URL of the control plane's Signaling API endpoint", + "format": "uri", + "examples": ["https://dataplane.example.com/api/v1/signaling"] + }, + "authorization": { + "$ref": "https://w3id.org/dspace-sig/v1.0/AuthorizationEntry.schema.json", + "description": "an authorization object" + } + } +} diff --git a/pkg/dsdk/testdata/schemas/DataFlowBaseMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowBaseMessage.schema.json new file mode 100644 index 0000000..617c1b9 --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowBaseMessage.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataFlowBaseMessage", + "type": "object", + "properties": { + "messageId": { + "type": "string", + "description": "Unique identifier for the message" + }, + "participantId": { + "type": "string", + "description": "ID of the participant sending the message" + }, + "counterPartyId": { + "type": "string", + "description": "ID of the counterparty" + }, + "dataspaceContext": { + "type": "string", + "description": "Context identifier of the dataspace" + }, + "dataFlowId": { + "type": "string", + "description": "The data flow identifier. This MUST equal the transfer process ID assigned by the control plane." + }, + "agreementId": { + "type": "string", + "description": "Contract Agreement identifier" + }, + "datasetId": { + "type": "string", + "description": "Dataset identifier. This should be identical to the @id of the dcat:dataset" + }, + "profile": { + "type": "string", + "description": "the flow transfer profile" + }, + "claims": { + "type": "object", + "description": "DSP claims of the counterparty as verified by the control plane" + }, + "metadata": { + "type": "object" + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "messageId", + "participantId", + "counterPartyId", + "dataspaceContext", + "dataFlowId", + "agreementId", + "datasetId", + "profile", + "claims" + ] +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataFlowPrepareMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowPrepareMessage.schema.json new file mode 100644 index 0000000..7b70f8d --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowPrepareMessage.schema.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "DataFlowPrepareMessage.json", + "title": "DataFlowPrepareMessage", + "allOf": [ + { + "$ref": "https://w3id.org/dspace-sig/v1.0/DataFlowBaseMessage.schema.json" + } + ] +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataFlowResumeMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowResumeMessage.schema.json new file mode 100644 index 0000000..2a06f7c --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowResumeMessage.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DataFlowResumeMessage", + "type": "object", + "properties": { + "messageId": { + "type": "string", + "description": "Unique identifier for the message" + }, + "dataAddress": { + "$ref": "https://w3id.org/dspace/2025/1/transfer/data-address-schema.json" + } + }, + "required": [ + "messageId" + ] +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataFlowStartMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowStartMessage.schema.json new file mode 100644 index 0000000..2cfb64c --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowStartMessage.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "DataFlowStartMessage.json", + "title": "DataFlowStartMessage", + "allOf": [ + { + "$ref": "https://w3id.org/dspace-sig/v1.0/DataFlowBaseMessage.schema.json" + } + ], + "properties": { + "dataAddress": { + "$ref": "https://w3id.org/dspace/2025/1/transfer/data-address-schema.json" + } + } +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataFlowStartedNotificationMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowStartedNotificationMessage.schema.json new file mode 100644 index 0000000..e21ba3f --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowStartedNotificationMessage.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "DataFlowStartedNotificationMessage.json", + "title": "DataFlowStartedNotificationMessage", + "type": "object", + "properties": { + "messageId": { + "type": "string", + "description": "Unique identifier for the message" + }, + "dataAddress": { + "$ref": "https://w3id.org/dspace/2025/1/transfer/data-address-schema.json" + } + }, + "required": ["messageId"] +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataFlowStatusMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowStatusMessage.schema.json new file mode 100644 index 0000000..f63147e --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowStatusMessage.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "DataFlowStatusMessage.json", + "title": "DataFlowStatusMessage", + "type": "object", + "properties": { + "messageId": { + "type": "string", + "description": "Unique identifier for the message" + }, + "dataFlowId": { + "type": "string" + }, + "dataAddress": { + "$ref": "https://w3id.org/dspace/2025/1/transfer/data-address-schema.json" + }, + "state": { + "type": "string", + "enum": [ + "INITIALIZED", + "PREPARING", + "PREPARED", + "STARTING", + "STARTED", + "COMPLETED", + "SUSPENDED", + "TERMINATED" + ] + }, + "error": { + "type": "string" + } + }, + "required": [ + "messageId", + "dataFlowId", + "state" + ] +} diff --git a/pkg/dsdk/testdata/schemas/DataFlowStatusResponseMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowStatusResponseMessage.schema.json new file mode 100644 index 0000000..de37350 --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowStatusResponseMessage.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "DataFlowStatusResponseMessage.json", + "title": "DataFlowStatusResponseMessage", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "INITIALIZED", + "PREPARING", + "PREPARED", + "STARTING", + "STARTED", + "COMPLETED", + "SUSPENDED", + "TERMINATED" + ], + "description": "The current state of the data flow" + }, + "dataFlowId": { + "type": "string", + "description": "The unique identifier of the data flow" + } + }, + "required": [ + "state", + "dataFlowId" + ] +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataFlowSuspendMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowSuspendMessage.schema.json new file mode 100644 index 0000000..828b745 --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowSuspendMessage.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "DataFlowSuspendMessage.json", + "title": "DataFlowSuspendMessage", + "type": "object", + "properties": { + "messageId": { + "type": "string", + "description": "Unique identifier for the message" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "messageId" + ] +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataFlowTerminateMessage.schema.json b/pkg/dsdk/testdata/schemas/DataFlowTerminateMessage.schema.json new file mode 100644 index 0000000..682b8dc --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataFlowTerminateMessage.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "DataFlowTerminateMessage.json", + "title": "DataFlowTerminateMessage", + "type": "object", + "properties": { + "messageId": { + "type": "string", + "description": "Unique identifier for the message" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "messageId" + ] +} \ No newline at end of file diff --git a/pkg/dsdk/testdata/schemas/DataPlaneRegistrationMessage.schema.json b/pkg/dsdk/testdata/schemas/DataPlaneRegistrationMessage.schema.json new file mode 100644 index 0000000..66d0a09 --- /dev/null +++ b/pkg/dsdk/testdata/schemas/DataPlaneRegistrationMessage.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "DataPlaneRegistrationMessage", + "required": ["dataplaneId", "endpoint", "profiles", "authorization"], + "properties": { + "dataplaneId": { + "type": "string", + "description": "The unique identifier of the data plane", + "examples": ["dataplane-64345"] + }, + "endpoint": { + "type": "string", + "description": "The base URL of the data plane's Signaling API endpoint", + "format": "uri", + "examples": ["https://dataplane.example.com/api/v1/signaling"] + }, + "profiles": { + "type": "array", + "description": "A list of transfer profiles supported by the data plane", + "items": { + "type": "string", + "description": "The flow transfer profile", + "examples": ["com.test.http-pull"] + }, + "examples": [ + ["com.test.http-pull", "com.test.mqtt-push-com.test.mqtt"] + ] + }, + "authorization": { + "$ref": "https://w3id.org/dspace-sig/v1.0/AuthorizationEntry.schema.json", + "description": "an authorization object" + }, + "labels": { + "type": "array", + "description": "an array of one of more strings corresponding to labels associated with the data plane", + "items": { + "type": "string" + }, + "examples": [ + ["label1", "label2"] + ] + } + } +} diff --git a/pkg/dsdk/testdata/schemas/dsp-data-address-schema.json b/pkg/dsdk/testdata/schemas/dsp-data-address-schema.json new file mode 100644 index 0000000..1abce9f --- /dev/null +++ b/pkg/dsdk/testdata/schemas/dsp-data-address-schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "title": "TransferSchema", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DataAddress" + } + ], + "$id": "https://w3id.org/dspace/2025/1/transfer/data-address-schema.json", + "definitions": { + "DataAddress": { + "type": "object", + "properties": { + "@type": { + "type": "string", + "const": "DataAddress" + }, + "endpointType": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "endpointProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/EndpointProperty" + }, + "minItems": 1 + } + }, + "required": [ + "@type", + "endpointType" + ] + }, + "EndpointProperty": { + "type": "object", + "properties": { + "@type": { + "type": "string", + "const": "EndpointProperty" + }, + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "@type", + "name", + "value" + ] + } + } +} \ No newline at end of file diff --git a/pkg/memory/memory.go b/pkg/memory/memory.go index 1d7ad78..f00271e 100644 --- a/pkg/memory/memory.go +++ b/pkg/memory/memory.go @@ -42,9 +42,8 @@ func (s *InMemoryStore) FindById(ctx context.Context, id string) (*dsdk.DataFlow return nil, dsdk.ErrNotFound } - // Return a copy to prevent external modifications - flowCopy := *flow - return &flowCopy, nil + // Return a deep copy to prevent external modifications + return flow.Copy(), nil } // Create creates a new DataFlow entry @@ -63,9 +62,8 @@ func (s *InMemoryStore) Create(ctx context.Context, flow *dsdk.DataFlow) error { return dsdk.ErrConflict } - // Store a copy to prevent external modifications - flowCopy := *flow - s.flows[flow.ID] = &flowCopy + // Store a deep copy to prevent external modifications + s.flows[flow.ID] = flow.Copy() return nil } @@ -85,9 +83,8 @@ func (s *InMemoryStore) Save(ctx context.Context, flow *dsdk.DataFlow) error { return dsdk.ErrNotFound } - // Store a copy to prevent external modifications - flowCopy := *flow - s.flows[flow.ID] = &flowCopy + // Store a deep copy to prevent external modifications + s.flows[flow.ID] = flow.Copy() return nil } diff --git a/pkg/memory/memory_test.go b/pkg/memory/memory_test.go index f5fe6de..49c3be5 100644 --- a/pkg/memory/memory_test.go +++ b/pkg/memory/memory_test.go @@ -438,3 +438,45 @@ func TestInMemoryStore_DataIsolation(t *testing.T) { assert.Equal(t, dsdk.Started, storedFlow.State) }) } + +func TestInMemoryStore_DeepCopiesNewFields(t *testing.T) { + store := NewInMemoryStore() + ctx := context.Background() + + flow := &dsdk.DataFlow{ + ID: "deep-copy", + Profile: "https://w3id.org/dspace-sig/profile/http-pull", + Claims: map[string]any{"membership": "active"}, + Labels: []string{"gold"}, + Metadata: map[string]any{"region": "westeurope"}, + ControlPlaneID: "cp-1", + SuspendedBy: dsdk.SuspendedByRemote, + SourceDataAddress: &dsdk.DataAddress{ + EndpointType: "https://w3id.org/idsa/v4.1/HTTP", + EndpointProperties: []dsdk.EndpointProperty{{Name: "authorization", Value: "token"}}, + }, + } + require.NoError(t, store.Create(ctx, flow)) + + // mutating the caller's instance must not change the stored flow + flow.Claims["membership"] = "revoked" + flow.Labels[0] = "changed" + flow.Metadata["region"] = "changed" + flow.SourceDataAddress.EndpointProperties[0].Value = "changed" + + found, err := store.FindById(ctx, "deep-copy") + require.NoError(t, err) + assert.Equal(t, "active", found.Claims["membership"]) + assert.Equal(t, []string{"gold"}, found.Labels) + assert.Equal(t, "westeurope", found.Metadata["region"]) + assert.Equal(t, "cp-1", found.ControlPlaneID) + assert.Equal(t, dsdk.SuspendedByRemote, found.SuspendedBy) + value, _ := found.SourceDataAddress.Property("authorization") + assert.Equal(t, "token", value) + + // mutating a returned instance must not change the stored flow either + found.Labels[0] = "changed" + again, err := store.FindById(ctx, "deep-copy") + require.NoError(t, err) + assert.Equal(t, []string{"gold"}, again.Labels) +} diff --git a/pkg/postgres/dataflow_schema.sql b/pkg/postgres/dataflow_schema.sql index 29ab059..a976bcb 100644 --- a/pkg/postgres/dataflow_schema.sql +++ b/pkg/postgres/dataflow_schema.sql @@ -5,31 +5,34 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -- Main table for DataFlow CREATE TABLE IF NOT EXISTS data_flows ( - id TEXT PRIMARY KEY NOT NULL, -- maps to DataFlow.ID - version BIGINT NOT NULL DEFAULT 0, -- DataFlow.Version (optimistic locking) - consumer BOOLEAN NOT NULL, -- DataFlow.Consumer - agreement_id TEXT NOT NULL, -- DataFlow.AgreementID - dataset_id TEXT NOT NULL, -- DataFlow.DatasetID - runtime_id TEXT NOT NULL, -- DataFlow.RuntimeID - participant_id TEXT NOT NULL, -- DataFlow.ParticipantID - dataspace_context TEXT NOT NULL, -- DataFlow.DataspaceContext - counterparty_id TEXT NOT NULL, -- DataFlow.CounterPartyID - - callback_address TEXT NOT NULL, -- DataFlow.CallbackAddress (URL as text) - - transfer_type_dest VARCHAR NOT NULL, -- DataFlow.TransferType {destinationType} - transfer_type_flowtype VARCHAR NOT NULL, -- DataFlow.TransferType {flowType} - source_data_address JSONB NOT NULL, -- DataFlow.SourceDataAddress {properties: {...}} - dest_data_address JSONB NOT NULL, -- DataFlow.DestinationDataAddress {properties: {...}} - - state INTEGER NOT NULL DEFAULT 0, -- DataFlow.State (enum int) - state_count INTEGER NOT NULL DEFAULT 0, -- DataFlow.StateCount - state_timestamp_ms BIGINT NOT NULL, -- DataFlow.StateTimestamp (epoch millis) - - error_detail VARCHAR, -- DataFlow.ErrorDetail - - created_at_ms BIGINT NOT NULL, -- DataFlow.CreatedAt (epoch millis) - updated_at_ms BIGINT NOT NULL -- DataFlow.UpdatedAt (epoch millis) + id TEXT PRIMARY KEY NOT NULL, -- maps to DataFlow.ID + version BIGINT NOT NULL DEFAULT 0, -- DataFlow.Version (optimistic locking) + consumer BOOLEAN NOT NULL, -- DataFlow.Consumer + agreement_id TEXT NOT NULL, -- DataFlow.AgreementID + dataset_id TEXT NOT NULL, -- DataFlow.DatasetID + runtime_id TEXT NOT NULL, -- DataFlow.RuntimeID + participant_id TEXT NOT NULL, -- DataFlow.ParticipantID + dataspace_context TEXT NOT NULL, -- DataFlow.DataspaceContext + counterparty_id TEXT NOT NULL, -- DataFlow.CounterPartyID + + profile TEXT NOT NULL, -- DataFlow.Profile (transfer profile, ends in -push or -pull) + claims JSONB, -- DataFlow.Claims + labels JSONB, -- DataFlow.Labels + metadata JSONB, -- DataFlow.Metadata + control_plane_id TEXT NOT NULL DEFAULT '', -- DataFlow.ControlPlaneID + suspended_by TEXT NOT NULL DEFAULT '', -- DataFlow.SuspendedBy ('', 'local' or 'remote') + + source_data_address JSONB, -- DataFlow.SourceDataAddress (DSP DataAddress) + dest_data_address JSONB, -- DataFlow.DestinationDataAddress (DSP DataAddress) + + state INTEGER NOT NULL DEFAULT 0, -- DataFlow.State (enum int) + state_count INTEGER NOT NULL DEFAULT 0, -- DataFlow.StateCount + state_timestamp_ms BIGINT NOT NULL, -- DataFlow.StateTimestamp (epoch millis) + + error_detail VARCHAR, -- DataFlow.ErrorDetail + + created_at_ms BIGINT NOT NULL, -- DataFlow.CreatedAt (epoch millis) + updated_at_ms BIGINT NOT NULL -- DataFlow.UpdatedAt (epoch millis) ); -- Helpful indexes @@ -38,8 +41,10 @@ CREATE INDEX IF NOT EXISTS idx_data_flows_updated_at ON data_flows (updated_at_m CREATE INDEX IF NOT EXISTS idx_data_flows_agreement ON data_flows (agreement_id); CREATE INDEX IF NOT EXISTS idx_data_flows_dataset ON data_flows (dataset_id); CREATE INDEX IF NOT EXISTS idx_data_flows_participant ON data_flows (participant_id); +CREATE INDEX IF NOT EXISTS idx_data_flows_control_plane ON data_flows (control_plane_id); -- Optional JSONB GIN indexes (uncomment if you need property-level queries) --- CREATE INDEX IF NOT EXISTS idx_data_flows_transfer_type_gin ON data_flows USING GIN (transfer_type); +-- CREATE INDEX IF NOT EXISTS idx_data_flows_labels_gin ON data_flows USING GIN (labels); +-- CREATE INDEX IF NOT EXISTS idx_data_flows_metadata_gin ON data_flows USING GIN (metadata); -- CREATE INDEX IF NOT EXISTS idx_data_flows_src_addr_gin ON data_flows USING GIN (source_data_address); --- CREATE INDEX IF NOT EXISTS idx_data_flows_dst_addr_gin ON data_flows USING GIN (dest_data_address); \ No newline at end of file +-- CREATE INDEX IF NOT EXISTS idx_data_flows_dst_addr_gin ON data_flows USING GIN (dest_data_address); diff --git a/pkg/postgres/postgres_store.go b/pkg/postgres/postgres_store.go index 2d3d7ee..be6df02 100644 --- a/pkg/postgres/postgres_store.go +++ b/pkg/postgres/postgres_store.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "log" + "reflect" "time" "github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/dsdk" @@ -22,12 +23,17 @@ func NewStore(db *sql.DB) *PostgresStore { return &PostgresStore{db: db} } +const columns = `id, version, consumer, agreement_id, dataset_id, runtime_id, participant_id, dataspace_context, + counterparty_id, profile, claims, labels, metadata, control_plane_id, suspended_by, source_data_address, + dest_data_address, state, state_count, state_timestamp_ms, error_detail, created_at_ms, updated_at_ms` + func (p PostgresStore) FindById(ctx context.Context, id string) (*dsdk.DataFlow, error) { - query := `SELECT * FROM data_flows WHERE id = $1` + query := `SELECT ` + columns + ` FROM data_flows WHERE id = $1` var df dsdk.DataFlow - var callbackAddressJson string - var sourceDataAddressJson, destDataAddressJson *string + var claims, labels, metadata, sourceDataAddress, destDataAddress []byte + var errorDetail sql.NullString + var suspendedBy string err := p.db.QueryRowContext(ctx, query, id).Scan( &df.ID, @@ -39,15 +45,18 @@ func (p PostgresStore) FindById(ctx context.Context, id string) (*dsdk.DataFlow, &df.ParticipantID, &df.DataspaceContext, &df.CounterPartyID, - &callbackAddressJson, - &df.TransferType.DestinationType, - &df.TransferType.FlowType, - &sourceDataAddressJson, - &destDataAddressJson, + &df.Profile, + &claims, + &labels, + &metadata, + &df.ControlPlaneID, + &suspendedBy, + &sourceDataAddress, + &destDataAddress, &df.State, &df.StateCount, &df.StateTimestamp, - &df.ErrorDetail, + &errorDetail, &df.CreatedAt, &df.UpdatedAt, ) @@ -58,19 +67,20 @@ func (p PostgresStore) FindById(ctx context.Context, id string) (*dsdk.DataFlow, if err != nil { return nil, err } + df.SuspendedBy = dsdk.SuspendedBy(suspendedBy) + df.ErrorDetail = errorDetail.String - if err := df.CallbackAddress.UnmarshalJSON([]byte(callbackAddressJson)); err != nil { - return nil, err - } - - if sourceDataAddressJson != nil { - if err := json.Unmarshal([]byte(*sourceDataAddressJson), &df.SourceDataAddress); err != nil { - return nil, err - } - } - - if destDataAddressJson != nil { - if err := json.Unmarshal([]byte(*destDataAddressJson), &df.DestinationDataAddress); err != nil { + for _, field := range []struct { + data []byte + target any + }{ + {claims, &df.Claims}, + {labels, &df.Labels}, + {metadata, &df.Metadata}, + {sourceDataAddress, &df.SourceDataAddress}, + {destDataAddress, &df.DestinationDataAddress}, + } { + if err := fromJson(field.data, field.target); err != nil { return nil, err } } @@ -84,7 +94,7 @@ func (p PostgresStore) Create(ctx context.Context, flow *dsdk.DataFlow) error { } query := ` INSERT INTO data_flows ( - id, + id, consumer, agreement_id, dataset_id, @@ -92,23 +102,24 @@ func (p PostgresStore) Create(ctx context.Context, flow *dsdk.DataFlow) error { participant_id, dataspace_context, counterparty_id, - callback_address, - transfer_type_dest, - transfer_type_flowtype, + profile, + claims, + labels, + metadata, + control_plane_id, + suspended_by, source_data_address, dest_data_address, state, + state_count, state_timestamp_ms, error_detail, created_at_ms, updated_at_ms - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)` + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22)` - cba, err := flow.CallbackAddress.MarshalJSON() - if err != nil { - return err - } - _, err = p.db.ExecContext(ctx, query, + now := time.Now().UnixMilli() + _, err := p.db.ExecContext(ctx, query, flow.ID, flow.Consumer, flow.AgreementID, @@ -117,16 +128,20 @@ func (p PostgresStore) Create(ctx context.Context, flow *dsdk.DataFlow) error { flow.ParticipantID, flow.DataspaceContext, flow.CounterPartyID, - cba, - flow.TransferType.DestinationType, - flow.TransferType.FlowType, + flow.Profile, + toJson(flow.Claims), + toJson(flow.Labels), + toJson(flow.Metadata), + flow.ControlPlaneID, + string(flow.SuspendedBy), toJson(flow.SourceDataAddress), toJson(flow.DestinationDataAddress), flow.State, - time.Now().UnixMilli(), + flow.StateCount, + now, flow.ErrorDetail, - time.Now().UnixMilli(), - time.Now().UnixMilli(), + now, + now, ) if err != nil { @@ -147,7 +162,7 @@ func (p PostgresStore) Save(ctx context.Context, flow *dsdk.DataFlow) error { // update query := ` UPDATE data_flows - SET + SET consumer = $1, agreement_id = $2, dataset_id = $3, @@ -155,16 +170,20 @@ func (p PostgresStore) Save(ctx context.Context, flow *dsdk.DataFlow) error { participant_id = $5, dataspace_context = $6, counterparty_id = $7, - callback_address = $8, - transfer_type_dest = $9, - transfer_type_flowtype = $10, - source_data_address = $11, - dest_data_address = $12, - state = $13, - state_timestamp_ms = $14, - error_detail = $15, - updated_at_ms = $16 - WHERE id = $17` + profile = $8, + claims = $9, + labels = $10, + metadata = $11, + control_plane_id = $12, + suspended_by = $13, + source_data_address = $14, + dest_data_address = $15, + state = $16, + state_count = $17, + state_timestamp_ms = $18, + error_detail = $19, + updated_at_ms = $20 + WHERE id = $21` _, err := p.db.ExecContext(ctx, query, flow.Consumer, @@ -174,12 +193,16 @@ func (p PostgresStore) Save(ctx context.Context, flow *dsdk.DataFlow) error { flow.ParticipantID, flow.DataspaceContext, flow.CounterPartyID, - toJson(flow.CallbackAddress), - flow.TransferType.DestinationType, - flow.TransferType.FlowType, + flow.Profile, + toJson(flow.Claims), + toJson(flow.Labels), + toJson(flow.Metadata), + flow.ControlPlaneID, + string(flow.SuspendedBy), toJson(flow.SourceDataAddress), toJson(flow.DestinationDataAddress), flow.State, + flow.StateCount, flow.StateTimestamp, flow.ErrorDetail, time.Now().UnixMilli(), @@ -208,7 +231,12 @@ func (p PostgresStore) Delete(ctx context.Context, id string) error { return nil } +// toJson marshals v into a JSONB value; nil maps, slices and pointers are stored as SQL NULL. func toJson(v any) *string { + rv := reflect.ValueOf(v) + if !rv.IsValid() || ((rv.Kind() == reflect.Map || rv.Kind() == reflect.Slice || rv.Kind() == reflect.Pointer) && rv.IsNil()) { + return nil + } j, err := json.Marshal(v) if err != nil { log.Fatalf("Failed to marshal: %v", err) @@ -218,6 +246,13 @@ func toJson(v any) *string { return &s } +func fromJson(data []byte, target any) error { + if len(data) == 0 { + return nil + } + return json.Unmarshal(data, target) +} + func exists(db *sql.DB, ctx context.Context, id string) bool { query := `SELECT COUNT(*) FROM data_flows WHERE id = $1` var count int diff --git a/pkg/postgres/postgres_store_test.go b/pkg/postgres/postgres_store_test.go index 2b2365a..ab643e4 100644 --- a/pkg/postgres/postgres_store_test.go +++ b/pkg/postgres/postgres_store_test.go @@ -171,10 +171,10 @@ func Test_FindById_WithDataAddress(t *testing.T) { id := uuid.New().String() err := store.Save(ctx, &dsdk.DataFlow{ ID: id, - SourceDataAddress: dsdk.DataAddress{ - Properties: map[string]any{ - "foo": "bar", - }, + SourceDataAddress: &dsdk.DataAddress{ + EndpointType: "https://w3id.org/idsa/v4.1/HTTP", + Endpoint: "https://example.com", + EndpointProperties: []dsdk.EndpointProperty{{Name: "foo", Value: "bar"}}, }, }) assert.NoError(t, err) @@ -182,7 +182,63 @@ func Test_FindById_WithDataAddress(t *testing.T) { found, err := store.FindById(ctx, id) assert.NoError(t, err) assert.Equal(t, id, found.ID) - assert.Equal(t, "bar", found.SourceDataAddress.Properties["foo"]) + value, ok := found.SourceDataAddress.Property("foo") + assert.True(t, ok) + assert.Equal(t, "bar", value) + assert.Nil(t, found.DestinationDataAddress) +} + +func Test_FindById_RoundTripsAllFields(t *testing.T) { + id := uuid.New().String() + flow := &dsdk.DataFlow{ + ID: id, + Consumer: true, + AgreementID: "agreement-id", + DatasetID: "dataset-id", + RuntimeID: "runtime-id", + ParticipantID: "participant-id", + DataspaceContext: "dataspace-context", + CounterPartyID: "counterparty-id", + Profile: "https://w3id.org/dspace-sig/profile/http-pull", + Claims: map[string]any{"membership": "active"}, + Labels: []string{"gold", "blue"}, + Metadata: map[string]any{"region": "westeurope"}, + ControlPlaneID: "control-plane-id", + SuspendedBy: dsdk.SuspendedByRemote, + State: dsdk.Suspended, + StateCount: 3, + ErrorDetail: "maintenance", + DestinationDataAddress: &dsdk.DataAddress{ + EndpointType: "https://w3id.org/idsa/v4.1/HTTP", + Endpoint: "https://example.com", + }, + } + assert.NoError(t, store.Create(ctx, flow)) + + found, err := store.FindById(ctx, id) + assert.NoError(t, err) + assert.Equal(t, flow.Profile, found.Profile) + assert.Equal(t, flow.Claims, found.Claims) + assert.Equal(t, flow.Labels, found.Labels) + assert.Equal(t, flow.Metadata, found.Metadata) + assert.Equal(t, flow.ControlPlaneID, found.ControlPlaneID) + assert.Equal(t, dsdk.SuspendedByRemote, found.SuspendedBy) + assert.Equal(t, dsdk.Suspended, found.State) + assert.Equal(t, uint(3), found.StateCount) + assert.Equal(t, "maintenance", found.ErrorDetail) + assert.Equal(t, flow.DestinationDataAddress, found.DestinationDataAddress) + assert.Nil(t, found.SourceDataAddress) + + found.SuspendedBy = dsdk.SuspendedByNone + found.State = dsdk.Started + found.Labels = nil + assert.NoError(t, store.Save(ctx, found)) + + updated, err := store.FindById(ctx, id) + assert.NoError(t, err) + assert.Equal(t, dsdk.SuspendedByNone, updated.SuspendedBy) + assert.Equal(t, dsdk.Started, updated.State) + assert.Nil(t, updated.Labels) } func Test_FindById_NotExists(t *testing.T) {