Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .github/workflows/ci_pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,23 @@ jobs:
go generate ./...
go test ./pkg/... ./internal/... --tags=postgres

tck:
name: Data Plane Signaling TCK
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true

- name: Run TCK
run: go test ./internal/tests/tck/... --tags=tck -v

verify-examples:
name: Run example ${{ matrix.example }}
runs-on: ubuntu-latest
Expand Down
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: all test test-postgres clean mock lint build help
.PHONY: all test test-postgres test-tck clean mock lint build help

# Go parameters
GOCMD=go
Expand Down Expand Up @@ -28,6 +28,9 @@ test: ## Run tests without postgres
test-postgres: ## Run all tests including postgres
$(GOTEST) -v ./... -tags postgres

test-tck: ## Run the Data Plane Signaling TCK against the SDK (needs Docker)
$(GOTEST) -v ./internal/tests/tck/... -tags tck

test-coverage: ## Run tests with coverage
$(GOTEST) -v -coverprofile=coverage.out ./...
$(GOCMD) tool cover -html=coverage.out -o coverage.html
Expand Down
11 changes: 7 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ Processors and handlers may modify the `DataFlow` they receive; the SDK persists
SDK fills in `messageId` and `dataFlowId`. A handler (`DataFlowHandler`) only returns an error, which aborts the
operation and leaves the data flow unchanged; the SDK then performs the transition itself. On `/started` the address
received from the provider (pull) is already on the flow (`SourceDataAddress`) when the handler runs, and the flow
always ends up STARTED. A re-delivered `/prepare` reaches the processor again with `ProcessorOptions.Duplicate` set;
always ends up STARTED. The specification makes that address optional, so the SDK accepts a `/started` without one;
a handler that needs it returns an error wrapping `ErrInvalidInput` (400). A re-delivered `/prepare` reaches the processor again with `ProcessorOptions.Duplicate` set;
re-delivered started, suspend, terminate and completed messages are acknowledged without invoking the handler.

- **Synchronous or asynchronous.** Report PREPARED or STARTED to complete the transition in the response (200).
Expand All @@ -109,9 +110,10 @@ re-delivered started, suspend, terminate and completed messages are acknowledged
(`.../http-push`, `.../http-pull`); see `DataFlow.IsPush` and `IsPull`.
- **Data addresses.** In a push transfer, the consumer hands out the address the provider pushes to (prepare response),
and the provider receives it with `/start`. In a pull transfer, the provider hands out the address the consumer pulls
from (start response), and the consumer receives it with `/started`. The SDK enforces these rules (400 otherwise),
drops addresses a processor returns where the specification allows none, and stores the addresses on the data flow
(`SourceDataAddress`, `DestinationDataAddress`).
from (start response), and the consumer receives it with `/started`. The SDK answers 400 to a `/start` or `/started`
that carries an address it must not carry and to a push `/start` without one, drops addresses a processor returns
where the specification allows none, and stores the addresses on the data flow (`SourceDataAddress`,
`DestinationDataAddress`).

## Control plane callbacks

Expand Down Expand Up @@ -214,4 +216,5 @@ also start an embedded NATS server.
go generate ./... # generates the mocks
go test ./... # unit and end-to-end tests
go test -tags postgres ./... # also the Postgres stores and integration tests; needs Docker (testcontainers)
go test -tags tck ./internal/tests/tck/... # the Data Plane Signaling TCK; needs Docker (testcontainers)
```
7 changes: 6 additions & 1 deletion examples/streaming-pull-dataplane/consumer/dataplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package consumer
import (
"context"
"errors"
"fmt"
"log"
"net/http"

Expand Down Expand Up @@ -76,8 +77,12 @@ func (d *ConsumerDataPlane) prepareProcessor(_ context.Context,
return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared}, nil
}

// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address.
// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address. The
// specification makes it optional, but this data plane cannot pull without it.
func (d *ConsumerDataPlane) startedProcessor(_ context.Context, flow *dsdk.DataFlow) error {
if flow.SourceDataAddress == nil {
return fmt.Errorf("%w: the started message contains no data address", dsdk.ErrInvalidInput)
}
endpoint := flow.SourceDataAddress.Endpoint
token, found := parseToken(natsservices.TokenKey, flow.SourceDataAddress)
if !found {
Expand Down
7 changes: 6 additions & 1 deletion examples/sync-pull-dataplane/consumer/dataplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"

Expand Down Expand Up @@ -92,8 +93,12 @@ func (d *ConsumerDataPlane) prepareProcessor(_ context.Context,
return &dsdk.DataFlowStatusMessage{State: dsdk.Prepared}, nil
}

// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address.
// startedProcessor runs on /started: the provider's DataAddress is on the flow as its source address. The
// specification makes it optional, but this data plane cannot pull without it.
func (d *ConsumerDataPlane) startedProcessor(_ context.Context, flow *dsdk.DataFlow) error {
if flow.SourceDataAddress == nil {
return fmt.Errorf("%w: the started message contains no data address", dsdk.ErrInvalidInput)
}
log.Printf("[Consumer Data Plane] Transfer access token available for participant %s dataset %s\n", flow.ParticipantID, flow.DatasetID)
endpoint := flow.SourceDataAddress.Endpoint
token, found := flow.SourceDataAddress.Property("token")
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.24.1

require (
github.com/MicahParks/keyfunc/v3 v3.8.0
github.com/docker/docker v28.3.3+incompatible
github.com/docker/go-connections v0.6.0
github.com/go-playground/validator/v10 v10.27.0
github.com/golang-jwt/jwt/v5 v5.3.1
Expand Down Expand Up @@ -33,7 +34,6 @@ require (
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/docker v28.3.3+incompatible // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.8.4 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand Down
5 changes: 4 additions & 1 deletion internal/tests/api_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,11 +229,14 @@ func Test_Started_Duplicate(t *testing.T) {
}

func Test_Started_DataAddressRules(t *testing.T) {
// the data address is optional for pull
pull := uuid.NewString()
createFlow(t, pull, pullProfile, dsdk.Prepared, true)
rr := post(t, "/v1/dataflows/"+pull+"/started", dsdk.DataFlowStartedNotificationMessage{MessageID: uuid.NewString()})
assert.Equal(t, http.StatusBadRequest, rr.Code)
assert.Equal(t, http.StatusOK, rr.Code)
assert.Equal(t, dsdk.Started, decodeStatus(t, rr).State)

// and must be omitted for push
push := uuid.NewString()
createFlow(t, push, pushProfile, dsdk.Prepared, true)
rr = post(t, "/v1/dataflows/"+push+"/started", dsdk.DataFlowStartedNotificationMessage{
Expand Down
167 changes: 167 additions & 0 deletions internal/tests/tck/dataplane_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
//go:build tck

package tck

import (
"context"
"fmt"
"sync"
"testing"
"time"

"github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/dsdk"
"github.com/eclipse-dataplane-core/dataplane-sdk-go/pkg/memory"
"github.com/stretchr/testify/require"
)

const (
controlPlaneID = "tck-control-plane"
tckAuthorizationType = "tck"

// agreementAsync is the agreement id of TCK tests that expect the data plane to prepare or start asynchronously.
agreementAsync = "async"
// agreementComplete is the agreement id of TCK tests that expect the data plane to complete the transfer.
agreementComplete = "complete"

// asyncDelay lets the SDK store the data flow, which it does after the processor returned, before the callback
// transitions it.
asyncDelay = 50 * time.Millisecond
// completeDelay gives the TCK time to see the data flow in STARTED, which it polls every 100ms, and to start
// listening for the completed callback.
completeDelay = 250 * time.Millisecond
)

// tckDataPlane is the data plane under test. It has no wire protocol: the processors only do what a TCK test asks
// for through the agreement id of its data flow.
type tckDataPlane struct {
sdk *dsdk.DataPlaneSDK

callbacks sync.WaitGroup
mu sync.Mutex
errors []error
}

// newDataPlane creates a data plane that sends its callbacks to the TCK at controlPlaneURL.
func newDataPlane(t *testing.T, controlPlaneURL string) *tckDataPlane {
dataPlane := &tckDataPlane{}
sdk, err := dsdk.NewDataPlaneSDK(
dsdk.WithStore(memory.NewInMemoryStore()),
dsdk.WithTransactionContext(memory.InMemoryTrxContext{}),
dsdk.WithAuthorization(tckAuthorization{}),
dsdk.WithControlPlane(dsdk.ControlPlane{
ID: controlPlaneID,
Endpoint: controlPlaneURL,
Authorization: dsdk.AuthorizationEntry{Type: tckAuthorizationType},
}),
dsdk.WithPrepareProcessor(dataPlane.prepare),
dsdk.WithStartProcessor(dataPlane.start),
dsdk.WithStartedProcessor(dataPlane.started),
)
require.NoError(t, err)
dataPlane.sdk = sdk
return dataPlane
}

// prepare is called on the consumer.
func (d *tckDataPlane) prepare(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) {
if flow.AgreementID != agreementAsync {
return status(flow, dsdk.Prepared), nil
}
id := flow.ID
d.callback(asyncDelay, "prepared", id, func(ctx context.Context) error {
_, err := d.sdk.NotifyPrepared(ctx, id, reportState(dsdk.Prepared))
return err
})
return &dsdk.DataFlowStatusMessage{State: dsdk.Preparing}, nil
}

// start is called on the provider.
func (d *tckDataPlane) start(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) {
id := flow.ID
switch flow.AgreementID {
case agreementAsync:
d.callback(asyncDelay, "started", id, func(ctx context.Context) error {
_, err := d.sdk.NotifyStarted(ctx, id, reportState(dsdk.Started))
return err
})
return &dsdk.DataFlowStatusMessage{State: dsdk.Starting}, nil
case agreementComplete:
d.complete(id)
}
return status(flow, dsdk.Started), nil
}

// started is called on the consumer.
func (d *tckDataPlane) started(_ context.Context, flow *dsdk.DataFlow) error {
if flow.AgreementID == agreementComplete {
d.complete(flow.ID)
}
return nil
}

func (d *tckDataPlane) complete(id string) {
d.callback(completeDelay, "completed", id, func(ctx context.Context) error {
return d.sdk.NotifyCompleted(ctx, id)
})
}

// callback sends a callback to the TCK after delay, and records its error.
func (d *tckDataPlane) callback(delay time.Duration, action string, id string, send func(ctx context.Context) error) {
d.callbacks.Add(1)
go func() {
defer d.callbacks.Done()
time.Sleep(delay)
if err := send(context.Background()); err != nil {
d.mu.Lock()
defer d.mu.Unlock()
d.errors = append(d.errors, fmt.Errorf("%s callback for data flow %s: %w", action, id, err))
}
}()
}

// wait blocks until all callbacks have been sent.
func (d *tckDataPlane) wait() {
d.callbacks.Wait()
}

// callbackErrors returns the errors of the callbacks that could not be sent or were rejected by the TCK.
func (d *tckDataPlane) callbackErrors() []error {
d.mu.Lock()
defer d.mu.Unlock()
return append([]error(nil), d.errors...)
}

// reportState returns a processor that reports the given state, with a DataAddress if the flow has to provide one.
func reportState(state dsdk.DataFlowState) dsdk.DataFlowProcessor {
return func(_ context.Context, flow *dsdk.DataFlow, _ *dsdk.DataPlaneSDK, _ *dsdk.ProcessorOptions) (*dsdk.DataFlowStatusMessage, error) {
return status(flow, state), nil
}
}

// status reports the given state. A push consumer and a pull provider hand out the DataAddress of their endpoint.
func status(flow *dsdk.DataFlow, state dsdk.DataFlowState) *dsdk.DataFlowStatusMessage {
message := &dsdk.DataFlowStatusMessage{State: state}
if (flow.Consumer && flow.IsPush()) || (!flow.Consumer && flow.IsPull()) {
message.DataAddress = &dsdk.DataAddress{EndpointType: "https://w3id.org/idsa/v4.1/HTTP", Endpoint: "http://localhost/data"}
}
return message
}

// tckAuthorization identifies the TCK by the token it sends with every request (tckToken), and sends the same token
// with the callbacks.
type tckAuthorization struct{}

func (tckAuthorization) Type() string {
return tckAuthorizationType
}

func (tckAuthorization) AuthorizationHeader(context.Context, dsdk.AuthorizationEntry) (string, error) {
return tckToken, nil
}

func (tckAuthorization) ExtractCallerID(_ context.Context, authorizationHeader string) (string, error) {
if authorizationHeader != tckToken {
return "", fmt.Errorf("%w: not the token of the TCK", dsdk.ErrUnauthorized)
}
return controlPlaneID, nil
}
19 changes: 19 additions & 0 deletions internal/tests/tck/dps.tck.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Configuration of the DPS TCK runtime (eclipsedataspacetck/dps-tck-runtime) for the data plane verification tests.
# The test passes the URL of the data plane under test and the authorization token as environment variables.

dataspacetck.test.package=org.eclipse.dataspacetck.dps.verification.dataplane
# the HTTP profile tests verify the HTTP wire protocol and its token renewal, which the SDK does not implement
dataspacetck.filters.tags.exclude=http-profile
dataspacetck.debug=true
dataspacetck.ansi=false

# The TCK cannot tell the data plane under test over HTTP how a test expects it to behave, so these tests signal it
# with a sentinel agreementId:
# "async" -> respond 202 with a transitional state, then send the prepared/started callback
# "complete" -> send the completed callback once the data flow is started
DP_C_PULL_03_01_AGREEMENTID=complete
DP_C_PULL_04_01_AGREEMENTID=async
DP_P_PULL_04_01_AGREEMENTID=async
DP_C_PUSH_04_01_AGREEMENTID=async
DP_P_PUSH_03_01_AGREEMENTID=complete
DP_P_PUSH_04_01_AGREEMENTID=async
Loading
Loading