From e79333b837e49433bc87bc5cf7649d4e3434cee6 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 10 Nov 2026 09:51:22 +0000 Subject: [PATCH] Deploy the testing property of pipelines and show remote changes in bundle plan bundle deploy sends a pipeline's testing on Create and Edit, and bundle plan reads it back, so a change made outside the bundle shows in the plan and the next deploy undoes it. The Go SDK does not have the field yet, so these calls are raw: a pipeline without testing is still created and edited with the SDK, and the raw read is used only with DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true. The test server stores spec.testing on Create and Edit and returns it on Get. Co-authored-by: Isaac --- .../bundle/refschema/output/pipelines.txt | 16 +-- .../resources/pipelines/testing/output.txt | 112 ++++++++++++++++++ .../bundle/resources/pipelines/testing/script | 33 ++++++ bundle/direct/dresources/pipeline.go | 86 ++++++++++++++ bundle/direct/dresources/pipeline_test.go | 75 ++++++++++++ bundle/direct/dresources/type_test.go | 4 - libs/testserver/fake_workspace.go | 2 + libs/testserver/pipelines.go | 9 ++ libs/testserver/pipelines_spec_testing.go | 45 +++++++ 9 files changed, 370 insertions(+), 12 deletions(-) create mode 100644 libs/testserver/pipelines_spec_testing.go diff --git a/acceptance/bundle/refschema/output/pipelines.txt b/acceptance/bundle/refschema/output/pipelines.txt index cee0902e460..9c2b4e56b5c 100644 --- a/acceptance/bundle/refschema/output/pipelines.txt +++ b/acceptance/bundle/refschema/output/pipelines.txt @@ -867,14 +867,14 @@ resources.pipelines.*.storage string ALL resources.pipelines.*.tags map[string]string ALL resources.pipelines.*.tags.* string ALL resources.pipelines.*.target string ALL -resources.pipelines.*.testing *resources.PipelineTesting INPUT -resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment INPUT -resources.pipelines.*.testing.environment.dependencies []string INPUT -resources.pipelines.*.testing.environment.dependencies[*] string INPUT -resources.pipelines.*.testing.sources []resources.PipelineTestSource INPUT -resources.pipelines.*.testing.sources[*] resources.PipelineTestSource INPUT -resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern INPUT -resources.pipelines.*.testing.sources[*].glob.include string INPUT +resources.pipelines.*.testing *resources.PipelineTesting ALL +resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment ALL +resources.pipelines.*.testing.environment.dependencies []string ALL +resources.pipelines.*.testing.environment.dependencies[*] string ALL +resources.pipelines.*.testing.sources []resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*] resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern ALL +resources.pipelines.*.testing.sources[*].glob.include string ALL resources.pipelines.*.trigger *pipelines.PipelineTrigger ALL resources.pipelines.*.trigger.cron *pipelines.CronTrigger ALL resources.pipelines.*.trigger.cron.quartz_cron_schedule string ALL diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index 02708b42ed8..60738208291 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -33,3 +33,115 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default Found 1 error + +=== Deploy sends testing, with the local wheel uploaded + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Created pipelines.orders +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "testing": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } +} + +=== No changes after the deploy + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A remote edit that drops testing shows in the plan + +>>> [CLI] bundle plan -o json +"update" + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Changing testing updates the pipeline + +>>> update_file.py databricks.yml ./tests/** ./tests/unit/** + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/unit/**" + } + } + ] +} + +=== Removing testing sends an update without it + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "testing": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index 245a0693110..99d5c998a7b 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -3,3 +3,36 @@ trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing title "Without pipeline testing, the property is an error\n" trace musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate + +title "Deploy sends testing, with the local wheel uploaded\n" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, testing: .body.testing}' + +title "No changes after the deploy\n" +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "A remote edit that drops testing shows in the plan\n" +PIPELINE_ID=$(read_id.py orders) +MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/pipelines/$PIPELINE_ID" | jq '.spec | del(.testing)' > edit.json +MSYS_NO_PATHCONV=1 $CLI api put "/api/2.0/pipelines/$PIPELINE_ID" --json @edit.json &> LOG.edit +rm edit.json +trace $CLI bundle plan -o json | jq '.plan["resources.pipelines.orders"].changes.testing.action' +trace $CLI bundle deploy +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "Changing testing updates the pipeline\n" +trace update_file.py databricks.yml './tests/**' './tests/unit/**' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, sources: .body.testing.sources}' + +title "Removing testing sends an update without it\n" +update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, testing: .body.testing}' + +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 6f367d4e498..3eacc3b5fa6 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -2,13 +2,19 @@ package dresources import ( "context" + "encoding/json" + "fmt" + "net/http" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -22,6 +28,9 @@ type PipelineState struct { // STs, Views). Nil means the server default (cascade) applies. Read from persisted state at // delete time; never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } func (s *PipelineState) UnmarshalJSON(b []byte) error { @@ -52,6 +61,9 @@ type PipelineRemote struct { PipelineId string `json:"pipeline_id,omitempty"` RunAsUserName string `json:"run_as_user_name,omitempty"` State pipelines.PipelineState `json:"state,omitempty"` + + // Testing is spec.testing of the Get response, read by DoRead because the Go SDK does not have it yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } // Custom marshaler needed because embedded CreatePipeline has its own MarshalJSON @@ -78,6 +90,7 @@ func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState return &PipelineState{ CreatePipeline: input.CreatePipeline, CascadeOnDestroy: input.CascadeOnDestroy, + Testing: input.Testing, } } @@ -86,10 +99,21 @@ func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { CreatePipeline: remote.CreatePipeline, // cascade_on_destroy is input-only and absent from PipelineRemote, so it stays nil here. CascadeOnDestroy: nil, + Testing: remote.Testing, } } +// pipelineTestingResponse holds spec.testing of a Get Pipeline response. +type pipelineTestingResponse struct { + Spec struct { + Testing *resources.PipelineTesting `json:"testing"` + } `json:"spec"` +} + func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { + if env.ExperimentalPipelineTesting(ctx) { + return r.doReadWithTesting(ctx, id) + } resp, err := r.client.Pipelines.GetByPipelineId(ctx, id) if err != nil { return nil, err @@ -97,6 +121,26 @@ func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemo return makePipelineRemote(resp), nil } +// doReadWithTesting sends the same request as GetByPipelineId, but raw, so that it can also read +// spec.testing, which GetPipelineResponse does not have yet. Remove it once GetPipelineResponse has it. +func (r *ResourcePipeline) doReadWithTesting(ctx context.Context, id string) (*PipelineRemote, error) { + var raw json.RawMessage + if err := r.doRaw(ctx, http.MethodGet, "/api/2.0/pipelines/"+id, nil, &raw); err != nil { + return nil, err + } + var resp pipelines.GetPipelineResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + var testing pipelineTestingResponse + if err := json.Unmarshal(raw, &testing); err != nil { + return nil, err + } + remote := makePipelineRemote(&resp) + remote.Testing = testing.Spec.Testing + return remote, nil +} + func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { var createPipeline pipelines.CreatePipeline if p.Spec != nil { @@ -155,10 +199,20 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { PipelineId: p.PipelineId, RunAsUserName: p.RunAsUserName, State: p.State, + // GetPipelineResponse has no spec.testing; DoRead sets it from the raw response. + Testing: nil, } } func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + if config.Testing != nil { + var response pipelines.CreatePipelineResponse + err := r.doWithTesting(ctx, http.MethodPost, "/api/2.0/pipelines", config.CreatePipeline, config.Testing, &response) + if err != nil { + return "", nil, err + } + return response.PipelineId, nil, nil + } response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err @@ -212,6 +266,9 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *Pipe ForceSendFields: utils.FilterFields[pipelines.EditPipeline](config.ForceSendFields), } + if config.Testing != nil { + return nil, r.doWithTesting(ctx, http.MethodPut, "/api/2.0/pipelines/"+id, request, config.Testing, nil) + } return nil, r.client.Pipelines.Update(ctx, request) } @@ -242,6 +299,35 @@ func (*ResourcePipeline) OverrideChangeDesc(_ context.Context, path *structpath. return nil } +// doWithTesting sends a Create or Edit request with the testing field, which the Go SDK does not +// have yet. The body is the SDK's JSON for the request plus testing. Remove this once +// CreatePipeline and EditPipeline have the field; pipelines without testing already use the SDK. +func (r *ResourcePipeline) doWithTesting(ctx context.Context, method, path string, request any, testing *resources.PipelineTesting, response any) error { + data, err := json.Marshal(request) + if err != nil { + return err + } + // RawMessage values keep the SDK's encoding, including large integers, unchanged. + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return err + } + body["testing"], err = json.Marshal(testing) + if err != nil { + return err + } + return r.doRaw(ctx, method, path, body, response) +} + +// doRaw sends a request that the Go SDK cannot express yet because it lacks the testing field. +func (r *ResourcePipeline) doRaw(ctx context.Context, method, path string, request, response any) error { + apiClient, err := client.New(r.client.Config) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + return apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response) +} + // Note, terraform provider either // a) reads back state at least once and fails create if state is "failed" // b) repeatededly reads state until state is "running" (if spec.Contionous is set). diff --git a/bundle/direct/dresources/pipeline_test.go b/bundle/direct/dresources/pipeline_test.go index 3629255a600..74f2b162f6f 100644 --- a/bundle/direct/dresources/pipeline_test.go +++ b/bundle/direct/dresources/pipeline_test.go @@ -4,8 +4,15 @@ import ( "reflect" "testing" + bundleenv "github.com/databricks/cli/bundle/env" + libsenv "github.com/databricks/cli/libs/env" + + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,3 +60,71 @@ func TestPipelineOverrideChangeDescCascadeOnDestroy(t *testing.T) { assert.Equal(t, deployplan.Skip, change.Action) }) } + +func TestPipelineRawRequests(t *testing.T) { + server := testserver.New(t) + var bodies, workspaceIDs []string + server.Handle("POST", "/api/2.0/pipelines", func(req testserver.Request) any { + bodies = append(bodies, string(req.Body)) + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return pipelines.CreatePipelineResponse{PipelineId: "p1"} + }) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return map[string]any{"pipeline_id": "p1", "spec": map[string]any{"name": "orders"}} + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token", WorkspaceID: "123"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + testingConfig := &resources.PipelineTesting{Sources: []resources.PipelineTestSource{{Glob: &pipelines.PathPattern{Include: "/Workspace/orders/tests/**"}}}} + // A false value in ForceSendFields shows that the raw body keeps the SDK's encoding. + create := pipelines.CreatePipeline{Name: "orders", Photon: false, ForceSendFields: []string{"Photon"}} + _, _, err = r.DoCreate(t.Context(), &PipelineState{CreatePipeline: create, Testing: testingConfig}) + require.NoError(t, err) + ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true") + _, err = r.DoRead(ctx, "p1") + require.NoError(t, err) + // Without testing, the request is the SDK's. + _, _, err = r.DoCreate(t.Context(), &PipelineState{Name: "orders"}) + require.NoError(t, err) + + // The raw requests carry the workspace ID, as the SDK's do. + assert.Equal(t, []string{"123", "123", "123"}, workspaceIDs) + assert.JSONEq(t, `{"name": "orders", "photon": false, "testing": {"sources": [{"glob": {"include": "/Workspace/orders/tests/**"}}]}}`, bodies[0]) + assert.JSONEq(t, `{"name": "orders"}`, bodies[1]) +} + +func TestPipelineReadReturnsTesting(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + return map[string]any{ + "pipeline_id": req.Vars["pipeline_id"], + "state": "IDLE", + "spec": map[string]any{ + "name": "orders", + "testing": map[string]any{"sources": []any{map[string]any{"glob": map[string]any{"include": "/Workspace/orders/tests/**"}}}}, + }, + } + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + // Without pipeline testing, the SDK reads the pipeline, as before the feature. + remote, err := r.DoRead(t.Context(), "p1") + require.NoError(t, err) + assert.Equal(t, "orders", remote.Name) + assert.Nil(t, remote.Testing) + + ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true") + remote, err = r.DoRead(ctx, "p1") + require.NoError(t, err) + assert.Equal(t, "p1", remote.PipelineId) + assert.Equal(t, "orders", remote.Name) + assert.Equal(t, pipelines.PipelineStateIdle, remote.State) + require.NotNil(t, remote.Testing) + require.Len(t, remote.Testing.Sources, 1) + assert.Equal(t, "/Workspace/orders/tests/**", remote.Testing.Sources[0].Glob.Include) + assert.Same(t, remote.Testing, r.RemapState(remote).Testing) +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index bf64ef100c4..d7515ae55a8 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -80,10 +80,6 @@ var knownMissingInStateType = map[string][]string{ "alerts": { "file_path", }, - "pipelines": { - // The experimental testing property is not deployed yet. - "testing", - }, "apps": { "lifecycle.prevent_destroy", "lifecycle.depends_on", diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c48ebb04df2..980f1367f43 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -223,6 +223,7 @@ type FakeWorkspace struct { JobRunIdempotency map[string]int64 Pipelines map[string]pipelines.GetPipelineResponse PipelineUpdates map[string]bool + pipelineTesting map[string]json.RawMessage Monitors map[string]catalog.MonitorInfo DataQualityMonitors map[string]dataquality.Monitor Apps map[string]apps.App @@ -532,6 +533,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Grants: map[string][]catalog.PrivilegeAssignment{}, Pipelines: map[string]pipelines.GetPipelineResponse{}, PipelineUpdates: map[string]bool{}, + pipelineTesting: map[string]json.RawMessage{}, Monitors: map[string]catalog.MonitorInfo{}, DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..97cf7073123 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -18,6 +18,13 @@ func (s *FakeWorkspace) PipelineGet(pipelineId string) Response { Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, } } + if testing, ok := s.pipelineTesting[pipelineId]; ok { + body, err := withPipelineTesting(value, testing) + if err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + return Response{Body: body} + } return Response{ Body: value, } @@ -80,6 +87,7 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = r + s.setPipelineTesting(pipelineId, req.Body) return Response{ Body: pipelines.CreatePipelineResponse{ @@ -139,6 +147,7 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response } setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = item + s.setPipelineTesting(pipelineId, req.Body) return Response{} } diff --git a/libs/testserver/pipelines_spec_testing.go b/libs/testserver/pipelines_spec_testing.go new file mode 100644 index 00000000000..ce397e5d8e4 --- /dev/null +++ b/libs/testserver/pipelines_spec_testing.go @@ -0,0 +1,45 @@ +package testserver + +import ( + "encoding/json" + + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +// pipelineTestingRequest holds the testing field of a Create or Edit request, which is not in the +// Go SDK yet. +type pipelineTestingRequest struct { + Testing json.RawMessage `json:"testing"` +} + +// setPipelineTesting records the testing field of a Create or Edit body, so that Get returns it. +// Like the rest of the spec, an Edit without testing removes it. +func (s *FakeWorkspace) setPipelineTesting(pipelineID string, body []byte) { + var request pipelineTestingRequest + _ = json.Unmarshal(body, &request) + if len(request.Testing) == 0 || string(request.Testing) == "null" { + delete(s.pipelineTesting, pipelineID) + return + } + s.pipelineTesting[pipelineID] = request.Testing +} + +// withPipelineTesting adds spec.testing to a Get Pipeline response, which the Go SDK type cannot +// hold. The JSON is merged as raw values, so numbers and the other fields are kept as they are. +func withPipelineTesting(value pipelines.GetPipelineResponse, testing json.RawMessage) (map[string]json.RawMessage, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, err + } + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return nil, err + } + var spec map[string]json.RawMessage + if err := json.Unmarshal(body["spec"], &spec); err != nil { + return nil, err + } + spec["testing"] = testing + body["spec"], err = json.Marshal(spec) + return body, err +}