From e5d9d2e3e3c172fe239ea766de3f1661fe315f6c Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger Date: Tue, 10 Nov 2026 14:41:00 +0100 Subject: [PATCH] feat!: run the DPS TCK and accept a pull /started without dataAddress Add an end-to-end test that runs the data plane verification tests of the Data Plane Signaling TCK (eclipsedataspacetck/dps-tck-runtime 1.3.3) against the SDK, behind the build tag "tck", with a CI job and a "make test-tck" target. The TCK sends a consumer-pull /started with only a messageId, which the SDK rejected with 400. The specification lists dataAddress as optional for this message, so Started now accepts it; a dataAddress on a push /started is still rejected. BREAKING CHANGE: a started handler can see a nil SourceDataAddress on a pull data flow. A handler that needs the address must check for it, e.g. by returning an error that wraps ErrInvalidInput (answered with 400), as the pull samples now do. Co-Authored-By: Claude --- .github/workflows/ci_pipeline.yml | 17 ++ Makefile | 5 +- README.md | 11 +- .../consumer/dataplane.go | 7 +- .../sync-pull-dataplane/consumer/dataplane.go | 7 +- go.mod | 2 +- internal/tests/api_integration_test.go | 5 +- internal/tests/tck/dataplane_test.go | 167 +++++++++++++++++ internal/tests/tck/dps.tck.properties | 19 ++ internal/tests/tck/tck_test.go | 168 ++++++++++++++++++ pkg/dsdk/dsdk.go | 20 +-- pkg/dsdk/dsdk_test.go | 44 +++-- pkg/dsdk/messages.go | 3 +- pkg/dsdk/router_test.go | 18 ++ 14 files changed, 456 insertions(+), 37 deletions(-) create mode 100644 internal/tests/tck/dataplane_test.go create mode 100644 internal/tests/tck/dps.tck.properties create mode 100644 internal/tests/tck/tck_test.go diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml index 8a48d5a..62cc847 100644 --- a/.github/workflows/ci_pipeline.yml +++ b/.github/workflows/ci_pipeline.yml @@ -39,6 +39,23 @@ jobs: go generate ./... go test ./pkg/... ./internal/... --tags=postgres + tck: + name: Data Plane Signaling TCK + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run TCK + run: go test ./internal/tests/tck/... --tags=tck -v + verify-examples: name: Run example ${{ matrix.example }} runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index b3578da..a4f8c51 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all test test-postgres clean mock lint build help +.PHONY: all test test-postgres test-tck clean mock lint build help # Go parameters GOCMD=go @@ -28,6 +28,9 @@ test: ## Run tests without postgres test-postgres: ## Run all tests including postgres $(GOTEST) -v ./... -tags postgres +test-tck: ## Run the Data Plane Signaling TCK against the SDK (needs Docker) + $(GOTEST) -v ./internal/tests/tck/... -tags tck + test-coverage: ## Run tests with coverage $(GOTEST) -v -coverprofile=coverage.out ./... $(GOCMD) tool cover -html=coverage.out -o coverage.html diff --git a/README.md b/README.md index e7f53aa..5a587b4 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,8 @@ Processors and handlers may modify the `DataFlow` they receive; the SDK persists SDK fills in `messageId` and `dataFlowId`. A handler (`DataFlowHandler`) only returns an error, which aborts the operation and leaves the data flow unchanged; the SDK then performs the transition itself. On `/started` the address received from the provider (pull) is already on the flow (`SourceDataAddress`) when the handler runs, and the flow -always ends up STARTED. A re-delivered `/prepare` reaches the processor again with `ProcessorOptions.Duplicate` set; +always ends up STARTED. The specification makes that address optional, so the SDK accepts a `/started` without one; +a handler that needs it returns an error wrapping `ErrInvalidInput` (400). A re-delivered `/prepare` reaches the processor again with `ProcessorOptions.Duplicate` set; re-delivered started, suspend, terminate and completed messages are acknowledged without invoking the handler. - **Synchronous or asynchronous.** Report PREPARED or STARTED to complete the transition in the response (200). @@ -109,9 +110,10 @@ re-delivered started, suspend, terminate and completed messages are acknowledged (`.../http-push`, `.../http-pull`); see `DataFlow.IsPush` and `IsPull`. - **Data addresses.** In a push transfer, the consumer hands out the address the provider pushes to (prepare response), and the provider receives it with `/start`. In a pull transfer, the provider hands out the address the consumer pulls - from (start response), and the consumer receives it with `/started`. The SDK enforces these rules (400 otherwise), - drops addresses a processor returns where the specification allows none, and stores the addresses on the data flow - (`SourceDataAddress`, `DestinationDataAddress`). + from (start response), and the consumer receives it with `/started`. The SDK answers 400 to a `/start` or `/started` + that carries an address it must not carry and to a push `/start` without one, drops addresses a processor returns + where the specification allows none, and stores the addresses on the data flow (`SourceDataAddress`, + `DestinationDataAddress`). ## Control plane callbacks @@ -214,4 +216,5 @@ also start an embedded NATS server. go generate ./... # generates the mocks go test ./... # unit and end-to-end tests go test -tags postgres ./... # also the Postgres stores and integration tests; needs Docker (testcontainers) +go test -tags tck ./internal/tests/tck/... # the Data Plane Signaling TCK; needs Docker (testcontainers) ``` diff --git a/examples/streaming-pull-dataplane/consumer/dataplane.go b/examples/streaming-pull-dataplane/consumer/dataplane.go index 916dc8e..78f8bd4 100644 --- a/examples/streaming-pull-dataplane/consumer/dataplane.go +++ b/examples/streaming-pull-dataplane/consumer/dataplane.go @@ -15,6 +15,7 @@ package consumer import ( "context" "errors" + "fmt" "log" "net/http" @@ -76,8 +77,12 @@ func (d *ConsumerDataPlane) prepareProcessor(_ context.Context, return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared}, nil } -// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address. +// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address. The +// specification makes it optional, but this data plane cannot pull without it. func (d *ConsumerDataPlane) startedProcessor(_ context.Context, flow *dsdk.DataFlow) error { + if flow.SourceDataAddress == nil { + return fmt.Errorf("%w: the started message contains no data address", dsdk.ErrInvalidInput) + } endpoint := flow.SourceDataAddress.Endpoint token, found := parseToken(natsservices.TokenKey, flow.SourceDataAddress) if !found { diff --git a/examples/sync-pull-dataplane/consumer/dataplane.go b/examples/sync-pull-dataplane/consumer/dataplane.go index 2db44f9..6c24a89 100644 --- a/examples/sync-pull-dataplane/consumer/dataplane.go +++ b/examples/sync-pull-dataplane/consumer/dataplane.go @@ -16,6 +16,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "log" "net/http" @@ -92,8 +93,12 @@ func (d *ConsumerDataPlane) prepareProcessor(_ context.Context, return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared}, nil } -// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address. +// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address. The +// specification makes it optional, but this data plane cannot pull without it. func (d *ConsumerDataPlane) startedProcessor(_ context.Context, flow *dsdk.DataFlow) error { + if flow.SourceDataAddress == nil { + return fmt.Errorf("%w: the started message contains no data address", dsdk.ErrInvalidInput) + } log.Printf("[Consumer Data Plane] Transfer access token available for participant %s dataset %s\n", flow.ParticipantID, flow.DatasetID) endpoint := flow.SourceDataAddress.Endpoint token, found := flow.SourceDataAddress.Property("token") diff --git a/go.mod b/go.mod index 621db31..cd87965 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.24.1 require ( github.com/MicahParks/keyfunc/v3 v3.8.0 + github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.6.0 github.com/go-playground/validator/v10 v10.27.0 github.com/golang-jwt/jwt/v5 v5.3.1 @@ -33,7 +34,6 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.3.3+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect diff --git a/internal/tests/api_integration_test.go b/internal/tests/api_integration_test.go index 891f19e..e87820f 100644 --- a/internal/tests/api_integration_test.go +++ b/internal/tests/api_integration_test.go @@ -229,11 +229,14 @@ func Test_Started_Duplicate(t *testing.T) { } func Test_Started_DataAddressRules(t *testing.T) { + // the data address is optional for pull pull := uuid.NewString() createFlow(t, pull, pullProfile, dsdk.Prepared, true) rr := post(t, "/v1/dataflows/"+pull+"/started", dsdk.DataFlowStartedNotificationMessage{MessageID: uuid.NewString()}) - assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Equal(t, http.StatusOK, rr.Code) + assert.Equal(t, dsdk.Started, decodeStatus(t, rr).State) + // and must be omitted for push push := uuid.NewString() createFlow(t, push, pushProfile, dsdk.Prepared, true) rr = post(t, "/v1/dataflows/"+push+"/started", dsdk.DataFlowStartedNotificationMessage{ diff --git a/internal/tests/tck/dataplane_test.go b/internal/tests/tck/dataplane_test.go new file mode 100644 index 0000000..7acd6cc --- /dev/null +++ b/internal/tests/tck/dataplane_test.go @@ -0,0 +1,167 @@ +//go:build tck + +package tck + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/dsdk" + "github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/memory" + "github.com/stretchr/testify/require" +) + +const ( + controlPlaneID = "tck-control-plane" + tckAuthorizationType = "tck" + + // agreementAsync is the agreement id of TCK tests that expect the data plane to prepare or start asynchronously. + agreementAsync = "async" + // agreementComplete is the agreement id of TCK tests that expect the data plane to complete the transfer. + agreementComplete = "complete" + + // asyncDelay lets the SDK store the data flow, which it does after the processor returned, before the callback + // transitions it. + asyncDelay = 50 * time.Millisecond + // completeDelay gives the TCK time to see the data flow in STARTED, which it polls every 100ms, and to start + // listening for the completed callback. + completeDelay = 250 * time.Millisecond +) + +// tckDataPlane is the data plane under test. It has no wire protocol: the processors only do what a TCK test asks +// for through the agreement id of its data flow. +type tckDataPlane struct { + sdk *dsdk.DataPlaneSDK + + callbacks sync.WaitGroup + mu sync.Mutex + errors []error +} + +// newDataPlane creates a data plane that sends its callbacks to the TCK at controlPlaneURL. +func newDataPlane(t *testing.T, controlPlaneURL string) *tckDataPlane { + dataPlane := &tckDataPlane{} + sdk, err := dsdk.NewDataPlaneSDK( + dsdk.WithStore(memory.NewInMemoryStore()), + dsdk.WithTransactionContext(memory.InMemoryTrxContext{}), + dsdk.WithAuthorization(tckAuthorization{}), + dsdk.WithControlPlane(dsdk.ControlPlane{ + ID: controlPlaneID, + Endpoint: controlPlaneURL, + Authorization: dsdk.AuthorizationEntry{Type: tckAuthorizationType}, + }), + dsdk.WithPrepareProcessor(dataPlane.prepare), + dsdk.WithStartProcessor(dataPlane.start), + dsdk.WithStartedProcessor(dataPlane.started), + ) + require.NoError(t, err) + dataPlane.sdk = sdk + return dataPlane +} + +// prepare is called on the consumer. +func (d *tckDataPlane) prepare(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { + if flow.AgreementID != agreementAsync { + return status(flow, dsdk.Prepared), nil + } + id := flow.ID + d.callback(asyncDelay, "prepared", id, func(ctx context.Context) error { + _, err := d.sdk.NotifyPrepared(ctx, id, reportState(dsdk.Prepared)) + return err + }) + return &dsdk.DataFlowStatusMessage{State: dsdk.Preparing}, nil +} + +// start is called on the provider. +func (d *tckDataPlane) start(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { + id := flow.ID + switch flow.AgreementID { + case agreementAsync: + d.callback(asyncDelay, "started", id, func(ctx context.Context) error { + _, err := d.sdk.NotifyStarted(ctx, id, reportState(dsdk.Started)) + return err + }) + return &dsdk.DataFlowStatusMessage{State: dsdk.Starting}, nil + case agreementComplete: + d.complete(id) + } + return status(flow, dsdk.Started), nil +} + +// started is called on the consumer. +func (d *tckDataPlane) started(_ context.Context, flow *dsdk.DataFlow) error { + if flow.AgreementID == agreementComplete { + d.complete(flow.ID) + } + return nil +} + +func (d *tckDataPlane) complete(id string) { + d.callback(completeDelay, "completed", id, func(ctx context.Context) error { + return d.sdk.NotifyCompleted(ctx, id) + }) +} + +// callback sends a callback to the TCK after delay, and records its error. +func (d *tckDataPlane) callback(delay time.Duration, action string, id string, send func(ctx context.Context) error) { + d.callbacks.Add(1) + go func() { + defer d.callbacks.Done() + time.Sleep(delay) + if err := send(context.Background()); err != nil { + d.mu.Lock() + defer d.mu.Unlock() + d.errors = append(d.errors, fmt.Errorf("%s callback for data flow %s: %w", action, id, err)) + } + }() +} + +// wait blocks until all callbacks have been sent. +func (d *tckDataPlane) wait() { + d.callbacks.Wait() +} + +// callbackErrors returns the errors of the callbacks that could not be sent or were rejected by the TCK. +func (d *tckDataPlane) callbackErrors() []error { + d.mu.Lock() + defer d.mu.Unlock() + return append([]error(nil), d.errors...) +} + +// reportState returns a processor that reports the given state, with a DataAddress if the flow has to provide one. +func reportState(state dsdk.DataFlowState) dsdk.DataFlowProcessor { + return func(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) { + return status(flow, state), nil + } +} + +// status reports the given state. A push consumer and a pull provider hand out the DataAddress of their endpoint. +func status(flow *dsdk.DataFlow, state dsdk.DataFlowState) *dsdk.DataFlowStatusMessage { + message := &dsdk.DataFlowStatusMessage{State: state} + if (flow.Consumer && flow.IsPush()) || (!flow.Consumer && flow.IsPull()) { + message.DataAddress = &dsdk.DataAddress{EndpointType: "https://w3id.org/idsa/v4.1/HTTP", Endpoint: "http://localhost/data"} + } + return message +} + +// tckAuthorization identifies the TCK by the token it sends with every request (tckToken), and sends the same token +// with the callbacks. +type tckAuthorization struct{} + +func (tckAuthorization) Type() string { + return tckAuthorizationType +} + +func (tckAuthorization) AuthorizationHeader(context.Context, dsdk.AuthorizationEntry) (string, error) { + return tckToken, nil +} + +func (tckAuthorization) ExtractCallerID(_ context.Context, authorizationHeader string) (string, error) { + if authorizationHeader != tckToken { + return "", fmt.Errorf("%w: not the token of the TCK", dsdk.ErrUnauthorized) + } + return controlPlaneID, nil +} diff --git a/internal/tests/tck/dps.tck.properties b/internal/tests/tck/dps.tck.properties new file mode 100644 index 0000000..0ab5b06 --- /dev/null +++ b/internal/tests/tck/dps.tck.properties @@ -0,0 +1,19 @@ +# Configuration of the DPS TCK runtime (eclipsedataspacetck/dps-tck-runtime) for the data plane verification tests. +# The test passes the URL of the data plane under test and the authorization token as environment variables. + +dataspacetck.test.package=org.eclipse.dataspacetck.dps.verification.dataplane +# the HTTP profile tests verify the HTTP wire protocol and its token renewal, which the SDK does not implement +dataspacetck.filters.tags.exclude=http-profile +dataspacetck.debug=true +dataspacetck.ansi=false + +# The TCK cannot tell the data plane under test over HTTP how a test expects it to behave, so these tests signal it +# with a sentinel agreementId: +# "async" -> respond 202 with a transitional state, then send the prepared/started callback +# "complete" -> send the completed callback once the data flow is started +DP_C_PULL_03_01_AGREEMENTID=complete +DP_C_PULL_04_01_AGREEMENTID=async +DP_P_PULL_04_01_AGREEMENTID=async +DP_C_PUSH_04_01_AGREEMENTID=async +DP_P_PUSH_03_01_AGREEMENTID=complete +DP_P_PUSH_04_01_AGREEMENTID=async diff --git a/internal/tests/tck/tck_test.go b/internal/tests/tck/tck_test.go new file mode 100644 index 0000000..dcd71b9 --- /dev/null +++ b/internal/tests/tck/tck_test.go @@ -0,0 +1,168 @@ +//go:build tck + +// Package tck runs the data plane verification tests of the Data Plane Signaling TCK +// (https://github.com/eclipse-dataspacetck/dps-tck) against the SDK. +package tck + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "regexp" + "slices" + "strconv" + "testing" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/dsdk" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + tckImage = "eclipsedataspacetck/dps-tck-runtime:1.3.3" + // tckCallbackPort is the port the TCK's control plane listens on for the data plane's callbacks. + tckCallbackPort = "8083" + // tckConfigFile is where the TCK runtime reads its configuration from. + tckConfigFile = "/etc/tck/config.properties" + // tckToken is the Authorization header value the TCK sends with every request. + tckToken = "tck-token" + // dockerHost is the name under which the TCK container reaches the data plane under test on the host. + dockerHost = "host.docker.internal" + tckTimeout = 5 * time.Minute +) + +// the TCK logs "Started: : " and then "SUCCESSFUL: " or "FAILED: " for every test +var ( + startedPattern = regexp.MustCompile(`Started: (\w+:\d+-\d+)`) + successfulPattern = regexp.MustCompile(`SUCCESSFUL: (\w+:\d+-\d+)`) +) + +// TestDataPlaneSignalingTCK starts a data plane on the host and runs the TCK, which acts as the control plane, in a +// container against it. The TCK reports its results on stdout only, so they are read from the container's log. +func TestDataPlaneSignalingTCK(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), tckTimeout) + defer cancel() + + // the data plane has to know its control plane before the TCK starts, so the callback port is reserved up front + callbackPort := freePort(t) + dataPlane := newDataPlane(t, "http://localhost:"+callbackPort) + + // listen on all interfaces: on Linux the container reaches the host through the Docker bridge + listener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + server := &http.Server{Handler: logFailedRequests(t, dsdk.NewHandler(dataPlane.sdk))} + go func() { _ = server.Serve(listener) }() + defer func() { _ = server.Close() }() + + dataPlaneURL := fmt.Sprintf("http://%s:%d%s", dockerHost, listener.Addr().(*net.TCPAddr).Port, dsdk.DefaultBasePath) + + tck, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: tckImage, + ExposedPorts: []string{callbackPort + ":" + tckCallbackPort + "/tcp"}, + Files: []testcontainers.ContainerFile{{ + HostFilePath: "dps.tck.properties", + ContainerFilePath: tckConfigFile, + FileMode: 0o644, + }}, + // the runtime reads every DATASPACETCK_* variable as the property dataspacetck.* + Env: map[string]string{ + "DATASPACETCK_DPS_DATAPLANE_URL": dataPlaneURL, + "DATASPACETCK_DPS_DATAPLANE_AUTHORIZATION": tckToken, + }, + HostConfigModifier: func(hostConfig *container.HostConfig) { + hostConfig.ExtraHosts = append(hostConfig.ExtraHosts, dockerHost+":host-gateway") + }, + // the runtime exits once all tests ran, with exit code 0 even if tests failed + WaitingFor: wait.ForExit().WithExitTimeout(tckTimeout), + }, + Started: true, + }) + if tck != nil { + defer func() { _ = tck.Terminate(context.Background()) }() + } + require.NoError(t, err) + + output := containerLog(t, ctx, tck) + t.Logf("TCK output:\n%s", output) + + dataPlane.wait() + require.Contains(t, output, "Test run complete") + started := matches(startedPattern, output) + require.NotEmpty(t, started, "the TCK did not run any test") + assert.Empty(t, without(started, matches(successfulPattern, output)), "TCK tests that were not successful") + assert.Empty(t, dataPlane.callbackErrors(), "failed callbacks to the TCK") +} + +// freePort returns a TCP port that is currently unused on the host. +func freePort(t *testing.T) string { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = listener.Close() }() + return strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) +} + +func containerLog(t *testing.T, ctx context.Context, c testcontainers.Container) string { + reader, err := c.Logs(ctx) + require.NoError(t, err) + defer func() { _ = reader.Close() }() + output, err := io.ReadAll(reader) + require.NoError(t, err) + return string(output) +} + +// matches returns the first capture group of every match of pattern in s. +func matches(pattern *regexp.Regexp, s string) []string { + var result []string + for _, match := range pattern.FindAllStringSubmatch(s, -1) { + result = append(result, match[1]) + } + return result +} + +// without returns the elements of all that are not in excluded. +func without(all []string, excluded []string) []string { + var result []string + for _, element := range all { + if !slices.Contains(excluded, element) { + result = append(result, element) + } + } + return result +} + +// logFailedRequests logs every request of the TCK that the data plane did not answer with a 2xx status. +func logFailedRequests(t *testing.T, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewReader(body)) + recorder := &responseRecorder{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(recorder, r) + if recorder.status >= http.StatusMultipleChoices { + t.Logf("%s %s\n request: %s\n response: %d %s", r.Method, r.URL.Path, body, recorder.status, recorder.body.String()) + } + }) +} + +type responseRecorder struct { + http.ResponseWriter + status int + body bytes.Buffer +} + +func (r *responseRecorder) WriteHeader(status int) { + r.status = status + r.ResponseWriter.WriteHeader(status) +} + +func (r *responseRecorder) Write(b []byte) (int, error) { + r.body.Write(b) + return r.ResponseWriter.Write(b) +} diff --git a/pkg/dsdk/dsdk.go b/pkg/dsdk/dsdk.go index 8cda100..5a66c8f 100644 --- a/pkg/dsdk/dsdk.go +++ b/pkg/dsdk/dsdk.go @@ -172,10 +172,12 @@ func (dsdk *DataPlaneSDK) Start(ctx context.Context, message DataFlowStartMessag return response, nil } -// Started is called on the consumer when the provider has started the transfer. For pull transfers the message must -// carry the DataAddress to obtain the data from; for push transfers it must not carry one. It records the received -// DataAddress on the flow (SourceDataAddress), invokes the started handler (WithStartedProcessor), transitions the -// flow to STARTED and persists it. A duplicate message for a STARTED flow is acknowledged without invoking the handler. +// Started is called on the consumer when the provider has started the transfer. For pull transfers the message +// carries the DataAddress to obtain the data from; the specification makes it optional, so the SDK accepts a message +// without one and leaves it to the started handler to reject it. For push transfers the message must not carry one. +// Started records the received DataAddress on the flow (SourceDataAddress), invokes the started handler +// (WithStartedProcessor), transitions the flow to STARTED and persists it. A duplicate message for a STARTED flow is +// acknowledged without invoking the handler. func (dsdk *DataPlaneSDK) Started(ctx context.Context, id string, message DataFlowStartedNotificationMessage) (*DataFlowStatusMessage, error) { if err := message.Validate(); err != nil { return nil, err @@ -193,10 +195,7 @@ func (dsdk *DataPlaneSDK) Started(ctx context.Context, id string, message DataFl if !flow.Consumer { return fmt.Errorf("%w: started is only valid for consumer data flows", ErrInvalidInput) } - switch { - case flow.IsPull() && message.DataAddress == nil: - return fmt.Errorf("%w: a started message for a pull transfer must contain a data address", ErrInvalidInput) - case flow.IsPush() && message.DataAddress != nil: + if flow.IsPush() && message.DataAddress != nil { return fmt.Errorf("%w: a started message for a push transfer must not contain a data address", ErrInvalidInput) } @@ -553,8 +552,9 @@ func WithStartProcessor(processor DataFlowProcessor) DataPlaneSDKOption { } // WithStartedProcessor sets the handler invoked for /started on the consumer, e.g. to start pulling data or to accept -// pushed data. For pull transfers the received DataAddress is on the flow (SourceDataAddress) when it runs. An error -// aborts the operation and leaves the flow unchanged; otherwise the flow transitions to STARTED. +// pushed data. For pull transfers the received DataAddress is on the flow (SourceDataAddress) when it runs, or nil if +// the message carried none; a handler that needs it returns an error wrapping ErrInvalidInput, which is answered with +// 400. An error aborts the operation and leaves the flow unchanged; otherwise the flow transitions to STARTED. func WithStartedProcessor(handler DataFlowHandler) DataPlaneSDKOption { return func(sdk *DataPlaneSDK) { sdk.onStarted = handler diff --git a/pkg/dsdk/dsdk_test.go b/pkg/dsdk/dsdk_test.go index edece23..3112f00 100644 --- a/pkg/dsdk/dsdk_test.go +++ b/pkg/dsdk/dsdk_test.go @@ -341,23 +341,33 @@ func Test_Started_Push(t *testing.T) { assert.Nil(t, response.DataAddress) } -func Test_Started_DataAddressRules(t *testing.T) { - tests := map[string]struct { - profile string - address *DataAddress - }{ - "pull without data address": {pullProfile, nil}, - "push with data address": {pushProfile, newDataAddress()}, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - sdk, store := newTestSDK(t) - store.put(consumerFlow("flow", tc.profile, Prepared)) - _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m", DataAddress: tc.address}) - assert.ErrorIs(t, err, ErrInvalidInput) - assert.Equal(t, Prepared, store.get(t, "flow").State) - }) - } +// Test_Started_PullWithoutDataAddress: the specification makes the data address of a started message optional, so +// the SDK accepts a pull transfer without one and the handler sees a nil address. +func Test_Started_PullWithoutDataAddress(t *testing.T) { + invoked := false + sdk, store := newTestSDK(t, WithStartedProcessor(func(_ context.Context, flow *DataFlow) error { + invoked = true + assert.Nil(t, flow.SourceDataAddress) + return nil + })) + store.put(consumerFlow("flow", pullProfile, Prepared)) + + response, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m"}) + require.NoError(t, err) + assert.Equal(t, Started, response.State) + assert.True(t, invoked) + + flow := store.get(t, "flow") + assert.Equal(t, Started, flow.State) + assert.Nil(t, flow.SourceDataAddress) +} + +func Test_Started_PushWithDataAddress(t *testing.T) { + sdk, store := newTestSDK(t) + store.put(consumerFlow("flow", pushProfile, Prepared)) + _, err := sdk.Started(context.Background(), "flow", DataFlowStartedNotificationMessage{MessageID: "m", DataAddress: newDataAddress()}) + assert.ErrorIs(t, err, ErrInvalidInput) + assert.Equal(t, Prepared, store.get(t, "flow").State) } func Test_Started_Duplicate(t *testing.T) { diff --git a/pkg/dsdk/messages.go b/pkg/dsdk/messages.go index 2ae5256..e5781fb 100644 --- a/pkg/dsdk/messages.go +++ b/pkg/dsdk/messages.go @@ -42,7 +42,8 @@ func (d *DataFlowStartMessage) Validate() error { } // DataFlowStartedNotificationMessage notifies the consumer data plane that a transfer has started. For consumer-pull -// transfers it carries the DataAddress to obtain the data from; it must be omitted for provider-push transfers. +// transfers it carries the DataAddress to obtain the data from, which the specification makes optional; it must be +// omitted for provider-push transfers. type DataFlowStartedNotificationMessage struct { MessageID string `json:"messageId" validate:"required"` DataAddress *DataAddress `json:"dataAddress,omitempty"` diff --git a/pkg/dsdk/router_test.go b/pkg/dsdk/router_test.go index e9ac359..4c602ed 100644 --- a/pkg/dsdk/router_test.go +++ b/pkg/dsdk/router_test.go @@ -1,7 +1,9 @@ package dsdk import ( + "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -105,6 +107,22 @@ func Test_Router_AcceptedSetsLocation(t *testing.T) { } } +// Test_Router_StartedHandlerRejectsMissingAddress: the SDK accepts a pull /started without a data address, so a +// handler that needs one rejects it with ErrInvalidInput. +func Test_Router_StartedHandlerRejectsMissingAddress(t *testing.T) { + sdk, store := newTestSDK(t, WithStartedProcessor(func(_ context.Context, flow *DataFlow) error { + if flow.SourceDataAddress == nil { + return fmt.Errorf("%w: no data address", ErrInvalidInput) + } + return nil + })) + store.put(consumerFlow("flow", pullProfile, Prepared)) + + rr := serve(t, NewHandler(sdk), http.MethodPost, "/v1/dataflows/flow/started", `{"messageId": "m"}`) + assert.Equal(t, http.StatusBadRequest, rr.Code, rr.Body.String()) + assert.Equal(t, Prepared, store.get(t, "flow").State) +} + func Test_Router_Errors(t *testing.T) { sdk, store := newTestSDK(t) handler := NewHandler(sdk)