Add DEV switch for DMS-authoritative deployments

acceptance/bundle/dms/history-only/databricks.yml.tmpl

@@ -0,0 +1,10 @@
+bundle:
+  name: dms-history-only-$UNIQUE_NAME
+
+experimental:
+  deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: history-only

acceptance/bundle/dms/history-only/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]

acceptance/bundle/dms/history-only/output.txt

@@ -0,0 +1,44 @@
+
+=== The DEV switch creates DMS-authoritative state
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-history-only-[UNIQUE_NAME]/default/files...
+Created jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> jq {features, state} .databricks/bundle/default/resources.json
+{
+  "features": {
+    "deployment_history": {}
+  },
+  "state": {}
+}
+
+=== The persisted state keeps DMS authoritative after the DEV switch is removed
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> print_requests.py --dms //resources --get --summary
+GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources
+
+=== DMS errors fail operations when DMS is authoritative
+>>> fault.py GET /api/2.0/bundle/deployments/* 500 0 1000
+
+>>> musterr [CLI] bundle plan
+Error: Fault injected by test. (500 INJECTED)
+
+Endpoint: GET [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]?
+HTTP Status: 500 Internal Server Error
+API error_code: INJECTED
+API message: Fault injected by test.
+
+
+>>> fault.py GET /api/2.0/bundle/deployments/* 500 0 0
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-history-only-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/dms/history-only/script

@@ -0,0 +1,21 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+title "The DEV switch creates DMS-authoritative state"
+trace $CLI bundle deploy
+trace jq '{features, state}' .databricks/bundle/default/resources.json
+rm -f "$OUT_REQUESTS"
+
+title "The persisted state keeps DMS authoritative after the DEV switch is removed"
+unset DATABRICKS_BUNDLE_DEV_DEPLOYMENT_HISTORY_ONLY
+rm -rf .databricks
+trace $CLI bundle plan
+trace print_requests.py --dms //resources --get --summary
+
+title "DMS errors fail operations when DMS is authoritative"
+trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 1000
+trace musterr $CLI bundle plan
+rm -f "$OUT_REQUESTS"
+
+trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 0
+trace $CLI bundle destroy --auto-approve
+rm -f "$OUT_REQUESTS"

acceptance/bundle/dms/history-only/test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+Env.DATABRICKS_BUNDLE_DEV_DEPLOYMENT_HISTORY_ONLY = "true"
+Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "0"

bundle/env/dms.go

@@ -7,6 +7,10 @@ import "context"
 // Deliberately undocumented.
 const DeploymentHistoryVariable = "DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY"
 
+// DevDeploymentHistoryOnlyVariable selects DMS-authoritative state for new deployments with history enabled.
+// Used for development and acceptance tests: https://github.com/databricks/cli/pull/7169.
+const DevDeploymentHistoryOnlyVariable = "DATABRICKS_BUNDLE_DEV_DEPLOYMENT_HISTORY_ONLY"
+
 // RecordsDeploymentHistory reports whether recording is on, from config or env var. Single predicate
 // for all recording code paths, keeping them in sync. Explicit config takes priority over the env var.
 // The env var only counts as "true": anything

cmd/bundle/utils/process.go

@@ -29,6 +29,7 @@ import (
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dms"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/logdiag"
 	"github.com/databricks/cli/libs/retry"
@@ -348,10 +349,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle
 		// Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.)
 		// A migrated-from-Terraform state is already open (seeded in memory above), so skip the disk open.
 		needDirectState := stateDesc.Engine.IsDirect() && needsState
+		historyOnly := usesDeploymentHistoryOnly(ctx, b, stateDesc)
 		var localPath string
 		if needDirectState && !b.DeploymentBundle.StateDB.IsOpen() {
 			_, localPath = b.StateFilenameDirect(ctx)
-			if !stateDesc.IsDMS() {
+			if !historyOnly {
 				if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil {
 					logdiag.LogError(ctx, err)
 					return b, stateDesc, root.ErrAlreadyPrinted
@@ -368,7 +370,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle
 
 		stateMode := dstate.WritingFile
 		switch {
-		case stateDesc.IsDMS():
+		case historyOnly:
 			// DMS-only state must keep its service-backed source.
 			stateMode = dstate.WritingDeploymentHistoryOnly
 		case opts.Deploy || opts.PreDeployChecks:
@@ -754,6 +756,14 @@ func configuresDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool {
 	return bundleenv.RecordsDeploymentHistory(ctx, configured)
 }
 
+// usesDeploymentHistoryOnly preserves existing state modes and allows opting new deployments into DMS-authoritative state.
+func usesDeploymentHistoryOnly(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) bool {
+	if stateDesc.IsDMS() {
+		return true
+	}
+	return stateDesc.SourcePath == "" && configuresDeploymentHistory(ctx, b) && env.Get(ctx, bundleenv.DevDeploymentHistoryOnlyVariable) == "true"
+}
+
 // isNewerVersion reports whether the state's recorded CLI version is strictly
 // newer than the running build. Both are bare versions without a leading "v".
 // An empty stateVersion (state not written by any CLI yet) or an unparseable

cmd/bundle/utils/process_test.go

@@ -3,11 +3,51 @@ package utils
 import (
 	"testing"
 
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/direct/dstate"
+	bundleenv "github.com/databricks/cli/bundle/env"
+	"github.com/databricks/cli/bundle/statemgmt"
+	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/databricks-sdk-go/service/bundledeployments"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func TestUsesDeploymentHistoryOnly(t *testing.T) {
+	for _, tt := range []struct {
+		name       string
+		configured *bool
+		historyEnv string
+		devEnv     string
+		sourcePath string
+		feature    string
+		want       bool
+	}{
+		{name: "default best effort", configured: new(true)},
+		{name: "config enabled", configured: new(true), devEnv: "true", want: true},
+		{name: "env enabled", historyEnv: "true", devEnv: "true", want: true},
+		{name: "history disabled", configured: new(false), historyEnv: "true", devEnv: "true"},
+		{name: "history unset", devEnv: "true"},
+		{name: "dev disabled", configured: new(true), devEnv: "false"},
+		{name: "dev invalid", configured: new(true), devEnv: "TRUE"},
+		{name: "existing file state", configured: new(true), devEnv: "true", sourcePath: "resources.json"},
+		{name: "existing shadow state", configured: new(true), devEnv: "true", sourcePath: "resources.json", feature: dstate.FeatureDeploymentHistoryShadow},
+		{name: "existing DMS state", sourcePath: "resources.json", feature: dstate.FeatureDeploymentHistory, want: true},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := env.Set(t.Context(), bundleenv.DeploymentHistoryVariable, tt.historyEnv)
+			ctx = env.Set(ctx, bundleenv.DevDeploymentHistoryOnlyVariable, tt.devEnv)
+			b := &bundle.Bundle{Config: config.Root{Experimental: &config.Experimental{DeploymentHistory: tt.configured}}}
+			state := &statemgmt.StateDesc{SourcePath: tt.sourcePath, Features: map[string]struct{}{}}
+			if tt.feature != "" {
+				state.Features[tt.feature] = struct{}{}
+			}
+			assert.Equal(t, tt.want, usesDeploymentHistoryOnly(ctx, b, state))
+		})
+	}
+}
+
 func TestIsNewerVersion(t *testing.T) {
 	tests := []struct {
 		name    string