Keep an explicit false on optional bundle config bools

.nextchanges/bundles/7412.md

@@ -0,0 +1 @@
+* Fixed an explicit `false` on optional boolean fields (`variables.*.sensitive`, `artifacts.*.dynamic_version`, `bundle.deployment.lock.force`, `workspace.azure_use_msi` and the `experimental.*` flags) being dropped, which hid it from `bundle validate -o json` and kept a target from overriding `true` with it. ([#7412](https://github.com/databricks/cli/pull/7412))

acceptance/bundle/refschema/output/root.txt

@@ -9,7 +9,7 @@ __locations.version	int	INPUT
 artifacts	config.Artifacts	INPUT
 artifacts.*	*config.Artifact	INPUT
 artifacts.*.build	string	INPUT
-artifacts.*.dynamic_version	bool	INPUT
+artifacts.*.dynamic_version	*bool	INPUT
 artifacts.*.executable	exec.ExecutableType	INPUT
 artifacts.*.files	[]config.ArtifactFile	INPUT
 artifacts.*.files[*]	config.ArtifactFile	INPUT
@@ -33,7 +33,7 @@ bundle.deployment.fail_on_active_runs	bool	INPUT
 bundle.deployment.latest_version_id	int	INPUT
 bundle.deployment.lock	config.Lock	INPUT
 bundle.deployment.lock.enabled	*bool	INPUT
-bundle.deployment.lock.force	bool	INPUT
+bundle.deployment.lock.force	*bool	INPUT
 bundle.engine	engine.EngineType	INPUT
 bundle.environment	string	INPUT
 bundle.force	bool	INPUT
@@ -55,7 +55,7 @@ environments.*	*config.Target	INPUT
 environments.*.artifacts	config.Artifacts	INPUT
 environments.*.artifacts.*	*config.Artifact	INPUT
 environments.*.artifacts.*.build	string	INPUT
-environments.*.artifacts.*.dynamic_version	bool	INPUT
+environments.*.artifacts.*.dynamic_version	*bool	INPUT
 environments.*.artifacts.*.executable	exec.ExecutableType	INPUT
 environments.*.artifacts.*.files	[]config.ArtifactFile	INPUT
 environments.*.artifacts.*.files[*]	config.ArtifactFile	INPUT
@@ -79,7 +79,7 @@ environments.*.bundle.deployment.fail_on_active_runs	bool	INPUT
 environments.*.bundle.deployment.latest_version_id	int	INPUT
 environments.*.bundle.deployment.lock	config.Lock	INPUT
 environments.*.bundle.deployment.lock.enabled	*bool	INPUT
-environments.*.bundle.deployment.lock.force	bool	INPUT
+environments.*.bundle.deployment.lock.force	*bool	INPUT
 environments.*.bundle.engine	engine.EngineType	INPUT
 environments.*.bundle.environment	string	INPUT
 environments.*.bundle.force	bool	INPUT
@@ -97,7 +97,7 @@ environments.*.bundle.terraform.exec_path	string	INPUT
 environments.*.bundle.uuid	string	INPUT
 environments.*.cluster_id	string	INPUT
 environments.*.compute_id	string	INPUT
-environments.*.default	bool	INPUT
+environments.*.default	*bool	INPUT
 environments.*.git	config.Git	INPUT
 environments.*.git.actual_branch	string	INPUT
 environments.*.git.branch	string	INPUT
@@ -148,7 +148,7 @@ environments.*.variables.*.lookup.pipeline	string	INPUT
 environments.*.variables.*.lookup.query	string	INPUT
 environments.*.variables.*.lookup.service_principal	string	INPUT
 environments.*.variables.*.lookup.warehouse	string	INPUT
-environments.*.variables.*.sensitive	bool	INPUT
+environments.*.variables.*.sensitive	*bool	INPUT
 environments.*.variables.*.type	variable.VariableType	INPUT
 environments.*.variables.*.value	any	INPUT
 environments.*.workspace	*config.Workspace	INPUT
@@ -159,7 +159,7 @@ environments.*.workspace.azure_client_id	string	INPUT
 environments.*.workspace.azure_environment	string	INPUT
 environments.*.workspace.azure_login_app_id	string	INPUT
 environments.*.workspace.azure_tenant_id	string	INPUT
-environments.*.workspace.azure_use_msi	bool	INPUT
+environments.*.workspace.azure_use_msi	*bool	INPUT
 environments.*.workspace.azure_workspace_resource_id	string	INPUT
 environments.*.workspace.client_id	string	INPUT
 environments.*.workspace.current_user	*config.User	INPUT
@@ -204,7 +204,7 @@ environments.*.workspace.current_user.schemas	[]iam.UserSchema	INPUT
 environments.*.workspace.current_user.schemas[*]	iam.UserSchema	INPUT
 environments.*.workspace.current_user.short_name	string	INPUT
 environments.*.workspace.current_user.userName	string	INPUT
-environments.*.workspace.experimental_is_unified_host	bool	INPUT
+environments.*.workspace.experimental_is_unified_host	*bool	INPUT
 environments.*.workspace.file_path	string	INPUT
 environments.*.workspace.google_service_account	string	INPUT
 environments.*.workspace.host	string	INPUT
@@ -217,19 +217,19 @@ environments.*.workspace.sync_mode	config.SyncMode	INPUT
 environments.*.workspace.workspace_id	string	INPUT
 experimental	*config.Experimental	INPUT
 experimental.deployment_history	*bool	INPUT
-experimental.immutable_folder	bool	INPUT
+experimental.immutable_folder	*bool	INPUT
 experimental.pydabs	config.PyDABs	INPUT
-experimental.pydabs.enabled	bool	INPUT
+experimental.pydabs.enabled	*bool	INPUT
 experimental.python	config.Python	INPUT
 experimental.python.mutators	[]string	INPUT
 experimental.python.mutators[*]	string	INPUT
 experimental.python.venv_path	string	INPUT
-experimental.python_wheel_wrapper	bool	INPUT
+experimental.python_wheel_wrapper	*bool	INPUT
 experimental.scripts	map[config.ScriptHook]config.Command	INPUT
 experimental.scripts.*	config.Command	INPUT
-experimental.skip_artifact_cleanup	bool	INPUT
-experimental.skip_name_prefix_for_schema	bool	INPUT
-experimental.use_legacy_run_as	bool	INPUT
+experimental.skip_artifact_cleanup	*bool	INPUT
+experimental.skip_name_prefix_for_schema	*bool	INPUT
+experimental.use_legacy_run_as	*bool	INPUT
 include	[]string	INPUT
 include[*]	string	INPUT
 permissions	[]resources.Permission	INPUT
@@ -272,7 +272,7 @@ targets.*	*config.Target	INPUT
 targets.*.artifacts	config.Artifacts	INPUT
 targets.*.artifacts.*	*config.Artifact	INPUT
 targets.*.artifacts.*.build	string	INPUT
-targets.*.artifacts.*.dynamic_version	bool	INPUT
+targets.*.artifacts.*.dynamic_version	*bool	INPUT
 targets.*.artifacts.*.executable	exec.ExecutableType	INPUT
 targets.*.artifacts.*.files	[]config.ArtifactFile	INPUT
 targets.*.artifacts.*.files[*]	config.ArtifactFile	INPUT
@@ -296,7 +296,7 @@ targets.*.bundle.deployment.fail_on_active_runs	bool	INPUT
 targets.*.bundle.deployment.latest_version_id	int	INPUT
 targets.*.bundle.deployment.lock	config.Lock	INPUT
 targets.*.bundle.deployment.lock.enabled	*bool	INPUT
-targets.*.bundle.deployment.lock.force	bool	INPUT
+targets.*.bundle.deployment.lock.force	*bool	INPUT
 targets.*.bundle.engine	engine.EngineType	INPUT
 targets.*.bundle.environment	string	INPUT
 targets.*.bundle.force	bool	INPUT
@@ -314,7 +314,7 @@ targets.*.bundle.terraform.exec_path	string	INPUT
 targets.*.bundle.uuid	string	INPUT
 targets.*.cluster_id	string	INPUT
 targets.*.compute_id	string	INPUT
-targets.*.default	bool	INPUT
+targets.*.default	*bool	INPUT
 targets.*.git	config.Git	INPUT
 targets.*.git.actual_branch	string	INPUT
 targets.*.git.branch	string	INPUT
@@ -365,7 +365,7 @@ targets.*.variables.*.lookup.pipeline	string	INPUT
 targets.*.variables.*.lookup.query	string	INPUT
 targets.*.variables.*.lookup.service_principal	string	INPUT
 targets.*.variables.*.lookup.warehouse	string	INPUT
-targets.*.variables.*.sensitive	bool	INPUT
+targets.*.variables.*.sensitive	*bool	INPUT
 targets.*.variables.*.type	variable.VariableType	INPUT
 targets.*.variables.*.value	any	INPUT
 targets.*.workspace	*config.Workspace	INPUT
@@ -376,7 +376,7 @@ targets.*.workspace.azure_client_id	string	INPUT
 targets.*.workspace.azure_environment	string	INPUT
 targets.*.workspace.azure_login_app_id	string	INPUT
 targets.*.workspace.azure_tenant_id	string	INPUT
-targets.*.workspace.azure_use_msi	bool	INPUT
+targets.*.workspace.azure_use_msi	*bool	INPUT
 targets.*.workspace.azure_workspace_resource_id	string	INPUT
 targets.*.workspace.client_id	string	INPUT
 targets.*.workspace.current_user	*config.User	INPUT
@@ -421,7 +421,7 @@ targets.*.workspace.current_user.schemas	[]iam.UserSchema	INPUT
 targets.*.workspace.current_user.schemas[*]	iam.UserSchema	INPUT
 targets.*.workspace.current_user.short_name	string	INPUT
 targets.*.workspace.current_user.userName	string	INPUT
-targets.*.workspace.experimental_is_unified_host	bool	INPUT
+targets.*.workspace.experimental_is_unified_host	*bool	INPUT
 targets.*.workspace.file_path	string	INPUT
 targets.*.workspace.google_service_account	string	INPUT
 targets.*.workspace.host	string	INPUT
@@ -449,7 +449,7 @@ variables.*.lookup.pipeline	string	INPUT
 variables.*.lookup.query	string	INPUT
 variables.*.lookup.service_principal	string	INPUT
 variables.*.lookup.warehouse	string	INPUT
-variables.*.sensitive	bool	INPUT
+variables.*.sensitive	*bool	INPUT
 variables.*.type	variable.VariableType	INPUT
 variables.*.value	any	INPUT
 workspace	config.Workspace	INPUT
@@ -460,7 +460,7 @@ workspace.azure_client_id	string	INPUT
 workspace.azure_environment	string	INPUT
 workspace.azure_login_app_id	string	INPUT
 workspace.azure_tenant_id	string	INPUT
-workspace.azure_use_msi	bool	INPUT
+workspace.azure_use_msi	*bool	INPUT
 workspace.azure_workspace_resource_id	string	INPUT
 workspace.client_id	string	INPUT
 workspace.current_user	*config.User	INPUT
@@ -505,7 +505,7 @@ workspace.current_user.schemas	[]iam.UserSchema	INPUT
 workspace.current_user.schemas[*]	iam.UserSchema	INPUT
 workspace.current_user.short_name	string	INPUT
 workspace.current_user.userName	string	INPUT
-workspace.experimental_is_unified_host	bool	INPUT
+workspace.experimental_is_unified_host	*bool	INPUT
 workspace.file_path	string	INPUT
 workspace.google_service_account	string	INPUT
 workspace.host	string	INPUT

acceptance/bundle/validate/explicit_false/databricks.yml

@@ -0,0 +1,43 @@
+bundle:
+  name: test-bundle
+  deployment:
+    lock:
+      force: true
+
+workspace:
+  azure_use_msi: true
+
+variables:
+  plain:
+    default: x
+    sensitive: false
+
+artifacts:
+  whl:
+    type: whl
+    build: "true"
+    dynamic_version: false
+  overridden:
+    type: whl
+    build: "true"
+    dynamic_version: true
+
+experimental:
+  python_wheel_wrapper: false
+  use_legacy_run_as: false
+  skip_artifact_cleanup: false
+  skip_name_prefix_for_schema: false
+
+targets:
+  dev:
+    default: true
+    # Each of these turns off what the root config turns on.
+    bundle:
+      deployment:
+        lock:
+          force: false
+    workspace:
+      azure_use_msi: false
+    artifacts:
+      overridden:
+        dynamic_version: false

acceptance/bundle/validate/explicit_false/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/explicit_false/output.txt

@@ -0,0 +1,27 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "lock_force": false,
+  "workspace": {
+    "azure_use_msi": false
+  },
+  "variables": {
+    "plain": {
+      "sensitive": false
+    }
+  },
+  "artifacts": {
+    "overridden": {
+      "dynamic_version": false
+    },
+    "whl": {
+      "dynamic_version": false
+    }
+  },
+  "experimental": {
+    "python_wheel_wrapper": false,
+    "skip_artifact_cleanup": false,
+    "skip_name_prefix_for_schema": false,
+    "use_legacy_run_as": false
+  }
+}

acceptance/bundle/validate/explicit_false/script

@@ -0,0 +1,9 @@
+# An explicit false is kept in the config, and a target can override true with false
+# (lock.force, azure_use_msi and artifacts.overridden.dynamic_version).
+trace $CLI bundle validate -o json | jq '{
+  lock_force: .bundle.deployment.lock.force,
+  workspace: .workspace | {azure_use_msi},
+  variables: .variables | map_values({sensitive}),
+  artifacts: .artifacts | map_values({dynamic_version}),
+  experimental
+}'

bundle/artifacts/build.go

@@ -79,7 +79,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 			}
 		}
 
-		if a.Type == "whl" && a.DynamicVersion && cacheDir != "" {
+		if a.Type == "whl" && config.IsExplicitlyEnabled(a.DynamicVersion) && cacheDir != "" {
 			b.Metrics.AddBoolValue(metrics.ArtifactDynamicVersionIsSet, true)
 			for ind, artifactFile := range a.Files {
 				patchedWheel := makePatchedWheel(ctx, cacheDir, artifactName, artifactFile.Source)

bundle/artifacts/upload.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/filer"
@@ -26,7 +27,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 		return diags
 	}
 
-	skipArtifactsCleanup := b.Config.Experimental != nil && b.Config.Experimental.SkipArtifactCleanup
+	skipArtifactsCleanup := b.Config.Experimental != nil && config.IsExplicitlyEnabled(b.Config.Experimental.SkipArtifactCleanup)
 	b.Metrics.AddBoolValue("skip_artifact_cleanup", skipArtifactsCleanup)
 	if skipArtifactsCleanup {
 		log.Info(ctx, "Skip cleaning up artifacts folder")

bundle/bundle.go

@@ -441,7 +441,7 @@ func (b *Bundle) SyncMode() config.SyncMode {
 	switch {
 	case b.Config.Workspace.SyncMode != "":
 		return b.Config.Workspace.SyncMode
-	case b.Config.Experimental != nil && b.Config.Experimental.ImmutableFolder:
+	case b.Config.Experimental != nil && config.IsExplicitlyEnabled(b.Config.Experimental.ImmutableFolder):
 		return config.SyncModeImmutable
 	case config.IsExplicitlyEnabled(sourceLinked):
 		return config.SyncModeInPlace

bundle/bundle_test.go

@@ -229,7 +229,7 @@ func TestBundleSyncMode(t *testing.T) {
 			b := &Bundle{Config: config.Root{
 				Workspace:    config.Workspace{SyncMode: tt.syncMode},
 				Presets:      config.Presets{SourceLinkedDeployment: tt.sourceLinked},
-				Experimental: &config.Experimental{ImmutableFolder: tt.immutable},
+				Experimental: &config.Experimental{ImmutableFolder: new(tt.immutable)},
 			}}
 			assert.Equal(t, tt.expected, b.SyncMode())
 			assert.Equal(t, tt.expected == config.SyncModeImmutable, b.IsImmutableFolder())

bundle/config/artifact.go

@@ -59,7 +59,7 @@ type Artifact struct {
 
 	Executable exec.ExecutableType `json:"executable,omitempty"`
 
-	DynamicVersion bool `json:"dynamic_version,omitempty"`
+	DynamicVersion *bool `json:"dynamic_version,omitempty"`
 
 	// Include narrows a `tgz` artifact to these subpaths of `path`, relative to it;
 	// empty packs the whole `path`. .gitignore is honored; the bundle-wide

bundle/config/experimental.go

@@ -10,15 +10,15 @@ type Experimental struct {
 	// path.
 	//
 	// ImmutableFolder is deprecated, use workspace.sync_mode: immutable instead.
-	ImmutableFolder bool `json:"immutable_folder,omitempty"`
+	ImmutableFolder *bool `json:"immutable_folder,omitempty"`
 
 	// By default Python wheel tasks deployed as is to Databricks platform.
 	// If notebook wrapper required (for example, used in DBR < 13.1 or other configuration differences), users can provide a following experimental setting
 	// experimental:
 	//    python_wheel_wrapper: true
 	// In this case the configured wheel task will be deployed as a notebook task which install defined wheel in runtime and executes it.
 	// For more details see https://github.com/databricks/cli/pull/797 and https://github.com/databricks/cli/pull/635
-	PythonWheelWrapper bool `json:"python_wheel_wrapper,omitempty"`
+	PythonWheelWrapper *bool `json:"python_wheel_wrapper,omitempty"`
 
 	// Enable legacy run_as behavior. That is:
 	// - Set the run_as identity as the owner of any pipelines in the bundle.
@@ -31,7 +31,7 @@ type Experimental struct {
 	// made in https://github.com/databricks/cli/pull/1233. This flag might
 	// be removed in the future once we have a proper workaround like allowing IS_OWNER
 	// as a top-level permission in the DAB.
-	UseLegacyRunAs bool `json:"use_legacy_run_as,omitempty"`
+	UseLegacyRunAs *bool `json:"use_legacy_run_as,omitempty"`
 
 	// PyDABs is deprecated use Python instead.
 	PyDABs PyDABs `json:"pydabs,omitempty"`
@@ -42,15 +42,15 @@ type Experimental struct {
 	// SkipArtifactCleanup determines whether to skip cleaning up the .internal folder
 	// containing build artifacts such as wheels. When set to true, the .internal folder
 	// and its contents will be preserved after bundle operations complete.
-	SkipArtifactCleanup bool `json:"skip_artifact_cleanup,omitempty"`
+	SkipArtifactCleanup *bool `json:"skip_artifact_cleanup,omitempty"`
 
 	// SkipNamePrefixForSchema skips adding the `presets.name_prefix` prefix
 	// to UC schemas defined in the bundle. Currently this is a opt-in behavior
 	// because turning it on by default will cause schema recreates and lose
 	// customer data.
 	// Eventually this can be made the default once we have native CRUD in DABs
 	// at which point we can deprecate or remove this field all together.
-	SkipNamePrefixForSchema bool `json:"skip_name_prefix_for_schema,omitempty"`
+	SkipNamePrefixForSchema *bool `json:"skip_name_prefix_for_schema,omitempty"`
 
 	// DeploymentHistory opts the bundle into the deployment metadata
 	// service (DMS), which records deployment history and tracks what changed
@@ -84,7 +84,7 @@ type Python struct {
 // PyDABs is deprecated use Python instead
 type PyDABs struct {
 	// Enabled is a flag to enable the feature.
-	Enabled bool `json:"enabled,omitempty"`
+	Enabled *bool `json:"enabled,omitempty"`
 }
 
 type (

bundle/config/lock.go

@@ -8,7 +8,7 @@ type Lock struct {
 
 	// Force acquisition of deployment lock even if it is currently held.
 	// This may be necessary if a prior deployment failed to release the lock.
-	Force bool `json:"force,omitempty"`
+	Force *bool `json:"force,omitempty"`
 }
 
 // IsEnabled checks if the deployment lock is enabled.

bundle/config/mutator/apply_sync_mode.go

@@ -121,7 +121,7 @@ func warnDeprecatedSyncMode(b *bundle.Bundle) diag.Diagnostics {
 		override = syncModePath
 	}
 
-	if b.Config.Experimental != nil && b.Config.Experimental.ImmutableFolder {
+	if b.Config.Experimental != nil && config.IsExplicitlyEnabled(b.Config.Experimental.ImmutableFolder) {
 		diags = diags.Append(deprecatedSyncModeWarning(b, immutableFolderPath, override))
 		if override == nil {
 			override = immutableFolderPath