diff --git a/acceptance/bundle/dms/history-only/databricks.yml.tmpl b/acceptance/bundle/dms/history-only/databricks.yml.tmpl new file mode 100644 index 00000000000..6327c3b64c7 --- /dev/null +++ b/acceptance/bundle/dms/history-only/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: dms-history-only-$UNIQUE_NAME + +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: history-only diff --git a/acceptance/bundle/dms/history-only/out.test.toml b/acceptance/bundle/dms/history-only/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/history-only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/history-only/output.txt b/acceptance/bundle/dms/history-only/output.txt new file mode 100644 index 00000000000..e13e05920f2 --- /dev/null +++ b/acceptance/bundle/dms/history-only/output.txt @@ -0,0 +1,44 @@ + +=== The DEV switch creates DMS-authoritative state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-history-only-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> jq {features, state} .databricks/bundle/default/resources.json +{ + "features": { + "deployment_history": {} + }, + "state": {} +} + +=== The persisted state keeps DMS authoritative after the DEV switch is removed +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //resources --get --summary +GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources + +=== DMS errors fail operations when DMS is authoritative +>>> fault.py GET /api/2.0/bundle/deployments/* 500 0 1000 + +>>> musterr [CLI] bundle plan +Error: Fault injected by test. (500 INJECTED) + +Endpoint: GET [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]? +HTTP Status: 500 Internal Server Error +API error_code: INJECTED +API message: Fault injected by test. + + +>>> fault.py GET /api/2.0/bundle/deployments/* 500 0 0 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-history-only-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/dms/history-only/script b/acceptance/bundle/dms/history-only/script new file mode 100644 index 00000000000..34a8fe36275 --- /dev/null +++ b/acceptance/bundle/dms/history-only/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "The DEV switch creates DMS-authoritative state" +trace $CLI bundle deploy +trace jq '{features, state}' .databricks/bundle/default/resources.json +rm -f "$OUT_REQUESTS" + +title "The persisted state keeps DMS authoritative after the DEV switch is removed" +unset DATABRICKS_BUNDLE_DEV_DEPLOYMENT_HISTORY_ONLY +rm -rf .databricks +trace $CLI bundle plan +trace print_requests.py --dms //resources --get --summary + +title "DMS errors fail operations when DMS is authoritative" +trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 1000 +trace musterr $CLI bundle plan +rm -f "$OUT_REQUESTS" + +trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 0 +trace $CLI bundle destroy --auto-approve +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/history-only/test.toml b/acceptance/bundle/dms/history-only/test.toml new file mode 100644 index 00000000000..aff0d1e56c8 --- /dev/null +++ b/acceptance/bundle/dms/history-only/test.toml @@ -0,0 +1,3 @@ +Cloud = false +Env.DATABRICKS_BUNDLE_DEV_DEPLOYMENT_HISTORY_ONLY = "true" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "0" diff --git a/bundle/env/dms.go b/bundle/env/dms.go index a92d9f09a07..78a5ef44921 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -7,6 +7,10 @@ import "context" // Deliberately undocumented. const DeploymentHistoryVariable = "DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY" +// DevDeploymentHistoryOnlyVariable selects DMS-authoritative state for new deployments with history enabled. +// Used for development and acceptance tests: https://github.com/databricks/cli/pull/7169. +const DevDeploymentHistoryOnlyVariable = "DATABRICKS_BUNDLE_DEV_DEPLOYMENT_HISTORY_ONLY" + // RecordsDeploymentHistory reports whether recording is on, from config or env var. Single predicate // for all recording code paths, keeping them in sync. Explicit config takes priority over the env var. // The env var only counts as "true": anything diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index c16f8280a08..4b7503ef1eb 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -29,6 +29,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/retry" @@ -348,10 +349,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) // A migrated-from-Terraform state is already open (seeded in memory above), so skip the disk open. needDirectState := stateDesc.Engine.IsDirect() && needsState + historyOnly := usesDeploymentHistoryOnly(ctx, b, stateDesc) var localPath string if needDirectState && !b.DeploymentBundle.StateDB.IsOpen() { _, localPath = b.StateFilenameDirect(ctx) - if !stateDesc.IsDMS() { + if !historyOnly { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -368,7 +370,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle stateMode := dstate.WritingFile switch { - case stateDesc.IsDMS(): + case historyOnly: // DMS-only state must keep its service-backed source. stateMode = dstate.WritingDeploymentHistoryOnly case opts.Deploy || opts.PreDeployChecks: @@ -754,6 +756,14 @@ func configuresDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { return bundleenv.RecordsDeploymentHistory(ctx, configured) } +// usesDeploymentHistoryOnly preserves existing state modes and allows opting new deployments into DMS-authoritative state. +func usesDeploymentHistoryOnly(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) bool { + if stateDesc.IsDMS() { + return true + } + return stateDesc.SourcePath == "" && configuresDeploymentHistory(ctx, b) && env.Get(ctx, bundleenv.DevDeploymentHistoryOnlyVariable) == "true" +} + // isNewerVersion reports whether the state's recorded CLI version is strictly // newer than the running build. Both are bare versions without a leading "v". // An empty stateVersion (state not written by any CLI yet) or an unparseable diff --git a/cmd/bundle/utils/process_test.go b/cmd/bundle/utils/process_test.go index 21910039344..3ff0f0f71c2 100644 --- a/cmd/bundle/utils/process_test.go +++ b/cmd/bundle/utils/process_test.go @@ -3,11 +3,51 @@ package utils import ( "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/direct/dstate" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestUsesDeploymentHistoryOnly(t *testing.T) { + for _, tt := range []struct { + name string + configured *bool + historyEnv string + devEnv string + sourcePath string + feature string + want bool + }{ + {name: "default best effort", configured: new(true)}, + {name: "config enabled", configured: new(true), devEnv: "true", want: true}, + {name: "env enabled", historyEnv: "true", devEnv: "true", want: true}, + {name: "history disabled", configured: new(false), historyEnv: "true", devEnv: "true"}, + {name: "history unset", devEnv: "true"}, + {name: "dev disabled", configured: new(true), devEnv: "false"}, + {name: "dev invalid", configured: new(true), devEnv: "TRUE"}, + {name: "existing file state", configured: new(true), devEnv: "true", sourcePath: "resources.json"}, + {name: "existing shadow state", configured: new(true), devEnv: "true", sourcePath: "resources.json", feature: dstate.FeatureDeploymentHistoryShadow}, + {name: "existing DMS state", sourcePath: "resources.json", feature: dstate.FeatureDeploymentHistory, want: true}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := env.Set(t.Context(), bundleenv.DeploymentHistoryVariable, tt.historyEnv) + ctx = env.Set(ctx, bundleenv.DevDeploymentHistoryOnlyVariable, tt.devEnv) + b := &bundle.Bundle{Config: config.Root{Experimental: &config.Experimental{DeploymentHistory: tt.configured}}} + state := &statemgmt.StateDesc{SourcePath: tt.sourcePath, Features: map[string]struct{}{}} + if tt.feature != "" { + state.Features[tt.feature] = struct{}{} + } + assert.Equal(t, tt.want, usesDeploymentHistoryOnly(ctx, b, state)) + }) + } +} + func TestIsNewerVersion(t *testing.T) { tests := []struct { name string