diff --git a/acceptance/bin/dms_resources.py b/acceptance/bin/dms_resources.py new file mode 100755 index 00000000000..25e469c5bad --- /dev/null +++ b/acceptance/bin/dms_resources.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Read resource ids and state from the deployment metadata service. + +While a bundle records deployment history the service owns the resource set, so the state file is +not where ids and state come from. The service is asked instead, which takes two lookups: the CLI +keeps no deployment id locally, and the id is the object id of the workspace node the service +registers under /resources.deployment.json (see libs/dms/resolve.go). +""" + +import functools +import glob +import json +import os +import posixpath +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from print_state import get_state_file + +CLI = os.environ.get("CLI", "databricks") + +# Must match dms.DeploymentNodeName. +DEPLOYMENT_NODE_NAME = "resources.deployment.json" + + +def run_json(cmd, allow_failure=False): + """Run cmd and parse its stdout, or return None if it fails and allow_failure is set. stderr is + captured rather than inherited: these lookups are plumbing, and a CLI warning like "no files to + sync" would otherwise land in the test output.""" + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") + if result.returncode != 0: + if allow_failure: + return None + raise SystemExit(f"{cmd} failed with code {result.returncode}\n{result.stdout}{result.stderr}".strip()) + return json.loads(result.stdout) + + +def records_deployment_history(): + """Whether this run records deployment history, so the service is what to ask.""" + return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" + + +def get_remote_state_path(target): + """The bundle's remote state directory. + + Preferred source is the sync snapshot, because it needs no CLI call: re-running the config + load would need whatever --var and flags the test deployed with, which a helper cannot know. + A bundle with no files to sync writes no snapshot, so fall back to asking the CLI - those + bundles are the ones with nothing to parameterize.""" + target_dir = os.path.dirname(get_state_file(target, False)) + snapshots = glob.glob(f"{target_dir}/sync-snapshots/*.json") + if snapshots: + # One snapshot per remote path, so a test that moved its root leaves several: the newest + # is the one the last deploy used. + newest = max(snapshots, key=os.path.getmtime) + remote_path = json.loads(open(newest).read())["remote_path"] + # state and files are siblings under the bundle root. + return posixpath.join(posixpath.dirname(remote_path), "state") + + args = [CLI, "bundle", "validate", "--output", "json"] + if target: + args += ["-t", target] + return run_json(args)["workspace"]["state_path"] + + +@functools.cache +def get_resources(target): + """Map every recorded resource key ("jobs.foo") to its {"id", "state"}. + + Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three + round trips and a script asks for one resource at a time. + """ + state_path = get_remote_state_path(target) + if not state_path: + return {} + + # No node means nothing has been recorded, the conclusion dms.resolveDeploymentID also draws + # from a 404 - the deployment is gone once the bundle is destroyed. + node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) + if not node or not node.get("object_id"): + return {} + deployment_id = node["object_id"] + + result = {} + # The service pages at 50 resources; the local fake returns everything at once. + page_token = None + while True: + url = f"/api/2.0/bundle/deployments/{deployment_id}/resources" + if page_token: + url += f"?page_token={page_token}" + listed = run_json([CLI, "api", "get", url]) + for resource in listed.get("resources") or []: + # The service stores state as the opaque envelope the CLI wrote (dstate.RecordedState), + # so unwrap it to the resource state itself. + envelope = json.loads(resource["state"]) if resource.get("state") else {} + result[resource["resource_key"]] = { + "id": resource.get("resource_id"), + "state": envelope.get("state") or {}, + "depends_on": envelope.get("depends_on") or [], + } + page_token = listed.get("next_page_token") + if not page_token: + return result diff --git a/acceptance/bin/nostamp b/acceptance/bin/nostamp deleted file mode 100755 index 1827400bece..00000000000 --- a/acceptance/bin/nostamp +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Reads JSON on stdin, writes it back with the deployment stamp removed. -# -# Deployment history recording adds deployment_id and version_id to every job and -# pipeline. Acceptance tests compare output byte for byte, so those two extra fields -# would fail every test in the DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true run -# (see bundle/test.toml). -# Pipe a plan, a state dump, or a resource payload through this and the test asserts one -# golden file either way. Tests under bundle/dms assert the stamp itself and must not. -# -# Three passes, because the stamp shows up in three shapes: -# -# 1. Nested in a deployment block, as printed by `jobs get` / `pipelines get`: -# "deployment": {"deployment_id": "87..", "kind": "BUNDLE", -# "metadata_file_path": "/x", "version_id": "1"} -# -> both keys dropped, "kind" and "metadata_file_path" kept. -# -# 2. Flat in a plan's "changes", keyed by field path: -# "changes": {"deployment.version_id": {"action": "skip", ...}, "name": {...}} -# -> the stamp entries dropped, real changes kept. -# -# 3. A "changes" object that pass 2 emptied to {} - it existed only because of -# recording, so the key goes too. -# -# Two things it deliberately keeps: -# -# - A deployment_id anywhere else. Pass 1 requires "kind" and "metadata_file_path" as -# neighbours, a pair unique to the deployment block, so an unrelated field of the -# same name is untouched. -# - "version_id": "". A terraform state dump carries that for a job it never stamped, -# and it is the test's own expected output, so `.value == ""` keeps it. -# -# Arguments are passed to jq, for callers whose input is not formatted the way jq -# formats by default: a state dump is printed verbatim from disk, so it needs -# --indent 1 to come back out unchanged. -jq "$@" '((.. | objects | select(has("kind") and has("metadata_file_path"))) - |= with_entries(select((.key | IN("deployment_id", "version_id")) == false or .value == ""))) - | ((.. | objects | .changes? | objects) - |= with_entries(select(.key | IN("deployment.deployment_id", "deployment.version_id") | not))) - | del(.. | objects | select(.changes == {}) | .changes)' diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py new file mode 100755 index 00000000000..a4306fe401a --- /dev/null +++ b/acceptance/bin/nostamp.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Read JSON on stdin, write it back with the DMS deployment stamp removed. + +Deployment history recording (DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true; see +bundle/test.toml) adds a deployment stamp to plans, states, and resource payloads. Pipe any of +those through this so an acceptance golden compares equal whether or not recording is on. Output +is a 2-space indent, keys in input order, <>& left unescaped, and integers at full precision. +Tests under bundle/dms assert the stamp itself and must not use it. + +Removed in three shapes plus the plan header: + 1. deployment_id/version_id nested in a deployment block, recognized by the + neighbouring "kind" and "metadata_file_path", which are kept; version_id + == "" is kept (a Terraform state dump carries that for an unstamped job). + 2. changes entries keyed "deployment.deployment_id" / "deployment.version_id". + 3. a changes object emptied by (2), or already empty. + 4. the recorded plan header fields deployment_id, next_version_id, last_version_id. +""" + +import argparse +import json +import sys + +_STAMP_KEYS = ("deployment_id", "version_id") +_CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id") +_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id") + + +def scrub(node): + if isinstance(node, dict): + # 1. A deployment block is the only object carrying both of these. + if "kind" in node and "metadata_file_path" in node: + for k in _STAMP_KEYS: + if node.get(k, "") != "": + node.pop(k, None) + out = {} + for k, v in node.items(): + if k == "changes" and isinstance(v, dict): + v = {ck: scrub(cv) for ck, cv in v.items() if ck not in _CHANGE_KEYS} + if not v: + continue # 3. drop a changes object left (or already) empty + out[k] = v + else: + out[k] = scrub(v) + return out + if isinstance(node, list): + return [scrub(x) for x in node] + return node + + +def render(data, indent): + data = scrub(data) + if isinstance(data, dict): + for k in _PLAN_HEADER_KEYS: + data.pop(k, None) # 4. plan header, present only at the root + return json.dumps(data, indent=indent, ensure_ascii=False, separators=(",", ": ")) + + +def main(): + parser = argparse.ArgumentParser() + # A state dump is printed with a single-space indent; plans use two. + parser.add_argument("--indent", type=int, default=2) + args = parser.parse_args() + + # Input is one JSON value (a plan or state dump) or a whitespace-separated stream of them (each + # request emitted by a print_requests filter) - jq accepted both here, so this must too. + text = sys.stdin.read() + decoder = json.JSONDecoder() + idx, n = 0, len(text) + while idx < n: + while idx < n and text[idx].isspace(): + idx += 1 + if idx >= n: + break + data, idx = decoder.raw_decode(text, idx) + sys.stdout.write(render(data, args.indent) + "\n") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 7e82f7b3d16..2959a76f81b 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -8,9 +8,14 @@ import argparse import glob +import json import os +def records_deployment_history(): + return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" + + def print_file(filename): data = open(filename).read() print(data, end="") @@ -53,6 +58,26 @@ def get_state_file(target, backup): return filtered[0] if filtered else result[0] +def print_recorded_state(filename, target): + """Print the state file with its resources filled in from the deployment metadata service. + + While recording, the file itself carries only the header - the service holds the resources - so + printing it raw would show an empty state and differ from the same test's non-recording run. + """ + # Imported here rather than at module level: dms_resources reads get_state_file from this module. + from dms_resources import get_resources + + data = json.loads(open(filename).read()) + state = {} + for key, value in sorted(get_resources(target).items()): + entry = {"__id__": value["id"], "state": value["state"]} + if value["depends_on"]: + entry["depends_on"] = value["depends_on"] + state[f"resources.{key}"] = entry + data["state"] = state + print(json.dumps(data, indent=1)) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("-t", "--target") @@ -60,7 +85,11 @@ def main(): args = parser.parse_args() for filename in get_state_files(args.target, args.backup): - if os.path.exists(filename): + if not os.path.exists(filename): + continue + if filename.endswith("resources.json") and records_deployment_history(): + print_recorded_state(filename, args.target) + else: print_file(filename) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 87cd2954bdf..06e7b69281e 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -15,6 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl +from dms_resources import get_resources, records_deployment_history from print_state import get_state_file @@ -34,6 +35,15 @@ def get_id_terraform(filename, name): print(f"Cannot find resource with {name=}. Available: {available}", file=sys.stderr) +def get_id_recorded(target, name): + resources = get_resources(target) + for key, value in resources.items(): + if key.split(".")[1] == name: + return value["id"] + + print(f"Cannot find recorded resource with {name=}. Available: {list(resources)}", file=sys.stderr) + + def get_id_direct(filename, name): raw = open(filename).read() data = json.loads(raw) @@ -53,11 +63,14 @@ def main(): parser.add_argument("name") args = parser.parse_args() - filename = get_state_file(args.target, args.backup) - if filename.endswith(".tfstate"): - id = get_id_terraform(filename, args.name) + if records_deployment_history(): + id = get_id_recorded(args.target, args.name) else: - id = get_id_direct(filename, args.name) + filename = get_state_file(args.target, args.backup) + if filename.endswith(".tfstate"): + id = get_id_terraform(filename, args.name) + else: + id = get_id_direct(filename, args.name) if id: print(id) diff --git a/acceptance/bin/read_state.py b/acceptance/bin/read_state.py index 0166bf9abbb..4d9bf84186e 100755 --- a/acceptance/bin/read_state.py +++ b/acceptance/bin/read_state.py @@ -9,6 +9,9 @@ import os import sys +sys.path.insert(0, os.path.dirname(__file__)) +from dms_resources import get_resources, records_deployment_history + def print_resource_terraform(group, name, *attrs): resource_type = "databricks_" + group[:-1] @@ -50,7 +53,21 @@ def print_resource_direct(group, name, *attrs): print(group, name, " ".join(values)) -if os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): +def print_resource_recorded(group, name, *attrs): + result = get_resources(None).get(f"{group}.{name}") + if result is None: + print(f"State not found for {group}.{name}") + return + + state = dict(result["state"]) + state.setdefault("id", result["id"]) + values = [f"{x}={state.get(x)!r}" for x in attrs] + print(group, name, " ".join(values)) + + +if records_deployment_history(): + print_resource_recorded(*sys.argv[1:]) +elif os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): print_resource_direct(*sys.argv[1:]) else: print_resource_terraform(*sys.argv[1:]) diff --git a/acceptance/bin/replace_ids.py b/acceptance/bin/replace_ids.py index e0ebf1cd0e6..2f5d8165032 100755 --- a/acceptance/bin/replace_ids.py +++ b/acceptance/bin/replace_ids.py @@ -10,6 +10,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl +from dms_resources import get_resources, records_deployment_history from print_state import get_state_file @@ -26,6 +27,12 @@ def iter_ids_terraform(filename): yield r_name, id +def iter_ids_recorded(target): + for key, value in get_resources(target).items(): + if value["id"]: + yield key.split(".")[1], value["id"] + + def iter_ids_direct(filename): raw = open(filename).read() data = json.loads(raw) @@ -44,11 +51,14 @@ def main(): parser.add_argument("--backup", action="store_true") args = parser.parse_args() - filename = get_state_file(args.target, args.backup) - if filename.endswith(".tfstate"): - it = iter_ids_terraform(filename) + if records_deployment_history(): + it = iter_ids_recorded(args.target) else: - it = iter_ids_direct(filename) + filename = get_state_file(args.target, args.backup) + if filename.endswith(".tfstate"): + it = iter_ids_terraform(filename) + else: + it = iter_ids_direct(filename) for name, id in it: add_repl(id, name.upper() + "_ID") diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json index c881a8a76b2..804ba480a90 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.test_job": { diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index b8d5ada22e7..ca1198e6750 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index ddb6fcd55ec..68d328cc67d 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -4,9 +4,9 @@ cp -r $TESTDIR/../whl_explicit/my_test_code . mkdir prebuilt cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt -trace $CLI bundle validate -o json | nostamp | jq .artifacts +trace $CLI bundle validate -o json | nostamp.py | jq .artifacts -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There are 2 original wheels and 2 patched ones" @@ -25,7 +25,7 @@ rm out.requests.txt title "Updating the local wheel and deploying again\n" touch my_test_code/src/new_module.py -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Verify contents, it should now have new_module.py" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 623849d3bb5..21bc2c9addc 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index f82c5d7eccf..d62de02530b 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in environments section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index ccc0d9ed7f8..59268fdeb0c 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting the environments dependency to keep its [train] extras suffix" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/bundle_tag/id/script b/acceptance/bundle/bundle_tag/id/script index 759cc89eba1..200c5a7d2e6 100644 --- a/acceptance/bundle/bundle_tag/id/script +++ b/acceptance/bundle/bundle_tag/id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/bundle_tag/url/script b/acceptance/bundle/bundle_tag/url/script index 759cc89eba1..200c5a7d2e6 100644 --- a/acceptance/bundle/bundle_tag/url/script +++ b/acceptance/bundle/bundle_tag/url/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/deploy/experimental-python/script b/acceptance/bundle/deploy/experimental-python/script index 09b90b28e2d..09bf4b14ef6 100644 --- a/acceptance/bundle/deploy/experimental-python/script +++ b/acceptance/bundle/deploy/experimental-python/script @@ -1,3 +1,3 @@ trace uv run --quiet --with $DATABRICKS_BUNDLES_WHEEL -- $CLI bundle deploy -trace $CLI jobs list --output json | nostamp +trace $CLI jobs list --output json | nostamp.py diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 85e9a908d1d..83136e53447 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -5,7 +5,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' trace $CLI bundle deploy diff --git a/acceptance/bundle/deploy/python-notebook/script b/acceptance/bundle/deploy/python-notebook/script index 21318c58b66..8b12727b59e 100644 --- a/acceptance/bundle/deploy/python-notebook/script +++ b/acceptance/bundle/deploy/python-notebook/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy -trace $CLI jobs list --output json | nostamp +trace $CLI jobs list --output json | nostamp.py diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_create.json b/acceptance/bundle/deploy/readplan/basic/out.plan_create.json index 4e8f66bafe4..c093dad07ab 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_create.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_create.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json index a6b4e835bb9..542dc55d6ef 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/deploy/readplan/basic/script b/acceptance/bundle/deploy/readplan/basic/script index 89bf2487b86..e01d5c5f3e7 100644 --- a/acceptance/bundle/deploy/readplan/basic/script +++ b/acceptance/bundle/deploy/readplan/basic/script @@ -1,5 +1,5 @@ # Generate initial plan -trace $CLI bundle plan -o json > out.plan_create.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "!POST" # First deploy to create the resource @@ -7,7 +7,7 @@ trace $CLI bundle deploy --plan out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "POST" # Generate a plan (should show "skip" since nothing changed) -trace $CLI bundle plan -o json > out.plan_skip.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.json trace print_requests.py --get //jobs | contains.py "GET" trace $CLI bundle deploy --plan out.plan_skip.json diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json index 932edf74f84..bd9561c9ec8 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "0.0.0-test", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json index 42cce15cbfc..80310520edb 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "0.0.0-test", "lineage": "different-lineage", "serial": 0, diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt index 387aac2335c..3ec4497a23e 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt @@ -1,4 +1,4 @@ -Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 2 +Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 3 Exit code: 1 diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml index e6ef3fcdcfb..576ff1ab001 100644 --- a/acceptance/bundle/deploy/readplan/test.toml +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -1,3 +1,3 @@ -# Saved plans don't carry the deployment stamp. Applying one plans an update on the next run. -# See dms_no_readplan in acceptance/bundle/test.toml. +# Dumps post-deploy state/plan, which carry the DMS deployment stamp under recording (applied at +# deploy, not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json b/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json index fa91f3d0bf9..84df30af6d8 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/unknown-field/script b/acceptance/bundle/deploy/readplan/unknown-field/script index 6032a3ae34e..ea17245ace1 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/script +++ b/acceptance/bundle/deploy/readplan/unknown-field/script @@ -1,5 +1,5 @@ # Generate a plan -trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.json # Modify the plan to rename "action" field to "action_renamed" # This will create an unknown field that the strict decoder should reject diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json index da6e6cae939..09a1645c786 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/script b/acceptance/bundle/deploy/wal/chain-3-jobs/script index c41e8e7f995..5cc77faa786 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/script +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/script @@ -7,7 +7,7 @@ trace errcode $CLI bundle deploy echo "" echo "=== WAL content after crash ===" -jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp || echo "No WAL file" +jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp.py || echo "No WAL file" echo "" echo "=== Number of jobs saved in WAL ===" diff --git a/acceptance/bundle/deploy/wal/crash-after-create/script b/acceptance/bundle/deploy/wal/crash-after-create/script index 5647c83e387..22f7587e52e 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/script +++ b/acceptance/bundle/deploy/wal/crash-after-create/script @@ -8,7 +8,7 @@ trace errcode $CLI bundle deploy trace assert_exists.py .databricks/bundle/default/resources.json.wal trace assert_not_exists.py .databricks/bundle/default/resources.json -trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp +trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp.py title "Any other command recovers state" $CLI bundle $COMMAND &> LOG.COMMAND.txt diff --git a/acceptance/bundle/deploy/wal/header-only-wal/output.txt b/acceptance/bundle/deploy/wal/header-only-wal/output.txt index 807fa926044..da8c1bdc1f3 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/output.txt +++ b/acceptance/bundle/deploy/wal/header-only-wal/output.txt @@ -28,7 +28,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> errcode assert_not_exists.py .databricks/bundle/default/resources.json.wal ->>> errcode cat .databricks/bundle/default/resources.json +>>> errcode print_state.py { "serial": 1, "state_keys": [ diff --git a/acceptance/bundle/deploy/wal/header-only-wal/script b/acceptance/bundle/deploy/wal/header-only-wal/script index 89da7e1a277..557fc7d4f47 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/script +++ b/acceptance/bundle/deploy/wal/header-only-wal/script @@ -19,4 +19,4 @@ title "Third deploy (must recover and succeed, not blocked by the leftover WAL)" trace errcode $CLI bundle deploy --force-lock trace errcode assert_not_exists.py .databricks/bundle/default/resources.json.wal -trace errcode cat .databricks/bundle/default/resources.json | jq -S '{serial: .serial, state_keys: (.state | keys)}' +trace errcode print_state.py | jq -S '{serial: .serial, state_keys: (.state | keys)}' diff --git a/acceptance/bundle/deployment/bind/experiment/script b/acceptance/bundle/deployment/bind/experiment/script index 70cf1933811..e31a17f7eaa 100644 --- a/acceptance/bundle/deployment/bind/experiment/script +++ b/acceptance/bundle/deployment/bind/experiment/script @@ -21,7 +21,7 @@ trace $CLI bundle deploy --force-lock --auto-approve # Use --output json to verify the change details including skip reason. # Write per-engine output since direct includes change details that terraform doesn't. # Filter to only the name change (tags/artifact_location may differ between local and cloud). -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' @@ -29,7 +29,7 @@ trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment. # Plan should still show no changes because the diff is suppressed. EXPERIMENT_NAME="//Users/${CURRENT_USER_NAME}/test-experiment$UNIQUE_NAME" envsubst < databricks.yml.tmpl > databricks.yml -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deployment unbind experiment1 diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index b02d93bcecb..2391f9d44e0 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -45,7 +45,7 @@ trace $CLI workspace get-status "${DEPLOYMENT_PATH}" | jq '{path, object_type}' title "Assert the pipeline is created" PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp | jq "{spec}" +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py | jq "{spec}" title "Assert the job is created:\n" JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.foo.id') diff --git a/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl new file mode 100644 index 00000000000..e909afaab5c --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: dms-declined-deploy-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_declined_deploy_schema_$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/dms/declined-deploy/out.requests.txt b/acceptance/bundle/dms/declined-deploy/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/declined-deploy/out.test.toml b/acceptance/bundle/dms/declined-deploy/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt new file mode 100644 index 00000000000..d09e8d18743 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -0,0 +1,88 @@ + +=== Deploy a schema, so the deployment and its first version exist +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-declined-deploy-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== A destructive change without --auto-approve is declined: this console cannot prompt +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 4 uploaded, 0 deleted + +=== Nothing was recorded for the declined deploy - no version, so none to abort +>>> print_requests.py --dms //api/2.0/bundle + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script new file mode 100644 index 00000000000..e67ff786b19 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/script @@ -0,0 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a schema, so the deployment and its first version exist" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "A destructive change without --auto-approve is declined: this console cannot prompt" +# Changing the catalog recreates the schema, which needs approval. +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace musterr $CLI bundle deploy + +title "Nothing was recorded for the declined deploy - no version, so none to abort" +# The version number it would have used is left for the next deploy to take, so the +# history has no entry that reads like a deploy which failed or did nothing. +trace print_requests.py --dms //api/2.0/bundle + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/depends-on/databricks.yml.tmpl b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl new file mode 100644 index 00000000000..1a60d215ecc --- /dev/null +++ b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: dms-depends-on-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + parent: + name: parent + child: + name: child + description: depends on ${resources.jobs.parent.id} diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt new file mode 100644 index 00000000000..4807497d63c --- /dev/null +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -0,0 +1,47 @@ + +=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... +Created jobs.child +Created jobs.parent +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/1/operations +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.parent", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.child + delete resources.jobs.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script new file mode 100644 index 00000000000..7f5576ce72b --- /dev/null +++ b/acceptance/bundle/dms/depends-on/script @@ -0,0 +1,8 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" +trace $CLI bundle deploy +trace print_requests.py --dms //versions/1/operations + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl new file mode 100644 index 00000000000..07f434b2e7a --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: dms-metadata-before-$UNIQUE_NAME +experimental: + record_deployment_history: true + +workspace: + # Pinned, so renaming the bundle or switching target below keeps the same deployment: the + # deployment is the workspace node under state_path, and a different path would be a + # different deployment. + state_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/dms-metadata-pinned-$UNIQUE_NAME/state + +resources: + schemas: + foo: + name: dms_metadata_schema_$UNIQUE_NAME + catalog_name: main + +targets: + dev: + default: true + prod: + mode: production diff --git a/acceptance/bundle/dms/deployment-metadata-change/out.test.toml b/acceptance/bundle/dms/deployment-metadata-change/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt new file mode 100644 index 00000000000..0b0798eee7f --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -0,0 +1,144 @@ + +=== Deploy: the deployment is created carrying this bundle's metadata +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-metadata-before-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-pinned-[UNIQUE_NAME]/state", + "target_name": "dev", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_metadata_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed +>>> update_file.py databricks.yml dms-metadata-before dms-metadata-after + +>>> [CLI] bundle deploy -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]", + "q": { + "update_mask": "display_name,target_name,deployment_mode,workspace_info" + }, + "body": { + "deployment_mode": "DEPLOYMENT_MODE_PRODUCTION", + "display_name": "dms-metadata-after-[UNIQUE_NAME]", + "target_name": "prod", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Deploy again without touching anything: the record already says this, so no deployment update +>>> [CLI] bundle deploy -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "2" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script new file mode 100644 index 00000000000..03f0c593999 --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -0,0 +1,14 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the deployment is created carrying this bundle's metadata" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed" +trace update_file.py databricks.yml dms-metadata-before dms-metadata-after +trace $CLI bundle deploy -t prod +trace print_requests.py --dms //api/2.0/bundle + +title "Deploy again without touching anything: the record already says this, so no deployment update" +trace $CLI bundle deploy -t prod +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl b/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl new file mode 100644 index 00000000000..b8c21a3cc96 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-emptied-resource-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_emptied_resource_$UNIQUE_NAME + catalog_name: main + grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] diff --git a/acceptance/bundle/dms/emptied-resource/out.test.toml b/acceptance/bundle/dms/emptied-resource/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt new file mode 100644 index 00000000000..d1170a2f40c --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -0,0 +1,97 @@ + +=== Deploy a schema with one grant: the grants node is recorded with its state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Created schemas.foo +Created schemas.foo.grants +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-emptied-resource-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + }, + { + "resource_key": "schemas.foo.grants", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_emptied_resource_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "schema/main.dms_emptied_resource_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource_[UNIQUE_NAME]\",\"__embed__\":[{\"principal\":\"deco-test-user@databricks.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Revoke the grant, so the grants node empties out +>>> update_file.py databricks.yml grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}] grants: [] +old='grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' not found in filename='databricks.yml' +bundle: + name: dms-emptied-resource-[UNIQUE_NAME] +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_emptied_resource_[UNIQUE_NAME] + catalog_name: main + grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] + + +Exit code: 1 diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script new file mode 100644 index 00000000000..5573dda9894 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a schema with one grant: the grants node is recorded with its state" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Revoke the grant, so the grants node empties out" +trace update_file.py databricks.yml 'grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' 'grants: []' +trace $CLI bundle deploy + +# The action type stays update: it is staged from the plan, before apply knows the state +# comes back empty. What drops the resource from the deployment is the update naming state +# with no value. +trace print_requests.py --dms //api/2.0/bundle + +title "Plan again: reading state back from the service works and reports no work" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl new file mode 100644 index 00000000000..29addd6433e --- /dev/null +++ b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: dms-existing-state-$UNIQUE_NAME + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/dms/existing-state/out.test.toml similarity index 58% rename from acceptance/bundle/state/feature_flags/test.toml rename to acceptance/bundle/dms/existing-state/out.test.toml index 5640bffa4dc..d73c45e3119 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/dms/existing-state/out.test.toml @@ -1,3 +1,3 @@ -Ignore = [".databricks"] - +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt new file mode 100644 index 00000000000..9a7439eef8a --- /dev/null +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -0,0 +1,78 @@ + +=== Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 2, + "features": null +} + +=== Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +=== The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +=== A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.one", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +>>> find.py resources[.]json[.]wal --expect 0 + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 2, + "features": { + "record_deployment_history": {} + } +} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script new file mode 100644 index 00000000000..e0784bb406b --- /dev/null +++ b/acceptance/bundle/dms/existing-state/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline +trace jq '{state_version, features}' .databricks/bundle/default/resources.json + +title "Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time" +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline + +title "The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace" +rm -rf .databricks +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy + +title "A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands" +mkdir -p .databricks/bundle/default +printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy + +title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed" +trace $CLI bundle destroy --auto-approve +printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/test.toml b/acceptance/bundle/dms/existing-state/test.toml new file mode 100644 index 00000000000..bc95f7e86c4 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/test.toml @@ -0,0 +1,3 @@ +# This test deploys without recording first, then turns it on per-command, so it opts out of +# the parent's DMS=true and drives DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY inline instead. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl new file mode 100644 index 00000000000..abdb86103ff --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-failed-delete-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + stuck: + name: stuck diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/dms/failed-delete/out.test.toml similarity index 68% rename from acceptance/bundle/state/feature_flags/out.test.toml rename to acceptance/bundle/dms/failed-delete/out.test.toml index 59b56a2037c..9921e91a794 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/dms/failed-delete/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt new file mode 100644 index 00000000000..701b2051c14 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -0,0 +1,149 @@ + +=== Deploy: the job is recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/files... +Created jobs.stuck +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-failed-delete-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.stuck", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.stuck", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone +>>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.stuck + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default + +Error: cannot delete resources.jobs.stuck: deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "jobs.stuck", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.stuck", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.stuck": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } +} diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script new file mode 100644 index 00000000000..fff56460c21 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/script @@ -0,0 +1,16 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the job is recorded" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" +# The state entry survives too, so the next deploy still knows about the resource. +trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle +trace print_state.py + +# print_state.py asks the service for the resources while recording, so drain those reads instead +# of leaving them behind for the harness to diff. +print_requests.py //api/2.0 > /dev/null diff --git a/acceptance/bundle/dms/failed-delete/test.toml b/acceptance/bundle/dms/failed-delete/test.toml new file mode 100644 index 00000000000..1a22ff04ef4 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - no job delete that the Jobs API refuses - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..a2937bf9a41 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-failed-recreate-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_recreate_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-recreate/out.requests.txt b/acceptance/bundle/dms/failed-recreate/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/failed-recreate/out.test.toml b/acceptance/bundle/dms/failed-recreate/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt new file mode 100644 index 00000000000..a012fec992f --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -0,0 +1,87 @@ + +=== Deploy the schema, so there is a resource to recreate +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the operation records no state, rather than describing the resource as it was before the deploy +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "sequence_id": "1", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The resource is not listed: state is what projects a resource, and the failed recreate left none +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{} + +=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists +>>> [CLI] bundle plan +create schemas.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script new file mode 100644 index 00000000000..cb00e0a0d8a --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/script @@ -0,0 +1,32 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the schema, so there is a resource to recreate" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the operation records no state, rather than describing the resource as it was before the deploy" +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +# Fail only the create that follows the delete; the delete itself still goes through. +trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py --dms //api/2.0/bundle + +title "The resource is not listed: state is what projects a resource, and the failed recreate left none" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/test.toml b/acceptance/bundle/dms/failed-recreate/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl new file mode 100644 index 00000000000..622598450cd --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: dms-failed-update-permissions-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + permissions: + - level: CAN_VIEW + user_name: viewer@example.com diff --git a/acceptance/bundle/dms/failed-update-permissions/out.test.toml b/acceptance/bundle/dms/failed-update-permissions/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt new file mode 100644 index 00000000000..080f4b6c8be --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -0,0 +1,29 @@ + +=== Deploy the job and its permissions +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created jobs.foo.permissions +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not +>>> update_file.py databricks.yml CAN_VIEW CAN_MANAGE + +>>> fault.py PUT /api/2.0/permissions/jobs/* 403 0 1 PERMISSION_DENIED + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... +Error: cannot update resources.jobs.foo.permissions: updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED) + +Endpoint: PUT [DATABRICKS_URL]/api/2.0/permissions/jobs/[NUMID] +HTTP Status: 403 Forbidden +API error_code: PERMISSION_DENIED +API message: Fault injected by test. + +Files: 3 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script new file mode 100644 index 00000000000..56141a30f4c --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -0,0 +1,12 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the job and its permissions" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not" +trace update_file.py databricks.yml "CAN_VIEW" "CAN_MANAGE" +# A 403 is what the API returns when the caller cannot change a job's owner. +trace fault.py "PUT /api/2.0/permissions/jobs/*" 403 0 1 PERMISSION_DENIED +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update-permissions/test.toml b/acceptance/bundle/dms/failed-update-permissions/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/failed-update/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl new file mode 100644 index 00000000000..87cfcff9f36 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-failed-update-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_update_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-update/out.test.toml b/acceptance/bundle/dms/failed-update/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt new file mode 100644 index 00000000000..125c16608b0 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -0,0 +1,86 @@ + +=== Deploy the schema, so there is an existing resource to update +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it +>>> update_file.py databricks.yml comment: v1 comment: v2 + +>>> fault.py PATCH /api/2.1/unity-catalog/schemas/* 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... +Error: cannot update resources.schemas.foo: updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.dms_failed_update_schema_[UNIQUE_NAME] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 3 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_UPDATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "name": "deployments/[NUMID]/resources/schemas.foo", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "resource_key": "schemas.foo", + "resource_type": "", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}" + } + ] +} + +=== Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys +>>> [CLI] bundle plan +update schemas.foo + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script new file mode 100644 index 00000000000..91640bd7631 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the schema, so there is an existing resource to update" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" +trace update_file.py databricks.yml "comment: v1" "comment: v2" +# Fail the update call itself, so the deploy never writes state for the schema. +trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py --dms //api/2.0/bundle + +title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-update/test.toml b/acceptance/bundle/dms/failed-update/test.toml new file mode 100644 index 00000000000..2b4eeb1bb04 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - no schema update that Unity Catalog rejects outright - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl new file mode 100644 index 00000000000..faa6ef5e63c --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: dms-multiple-resources-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt new file mode 100644 index 00000000000..ecc2092225a --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -0,0 +1,28 @@ + +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... +Created jobs.five +Created jobs.four +Created jobs.one +Created jobs.three +Created jobs.two +Files: 5 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/1/operations --sort --del-body state --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 5 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script new file mode 100644 index 00000000000..584423a4d5e --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +trace $CLI bundle deploy +trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline + +title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/no-drift/databricks.yml.tmpl b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl new file mode 100644 index 00000000000..e9892fa79a5 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: dms-no-drift-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + # Unique: the Pipelines API refuses a duplicate name, so two overlapping runs on the + # same workspace would collide. Jobs allow duplicates, which is why foo above does not. + name: bar-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + libraries: + - file: + path: ./transform.py diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt new file mode 100644 index 00000000000..e15952f27ae --- /dev/null +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -0,0 +1,63 @@ + +=== Deploy, then plan without touching anything: the deployment stamp must not show as drift +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created pipelines.bar +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== A second deploy is a no-op too: no update request for either resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script new file mode 100644 index 00000000000..ffe3976b227 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/script @@ -0,0 +1,12 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy, then plan without touching anything: the deployment stamp must not show as drift" +trace $CLI bundle deploy + +# Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, +# so it must ignore it or report spurious changes. +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +title "A second deploy is a no-op too: no update request for either resource" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort diff --git a/acceptance/bundle/dms/no-drift/transform.py b/acceptance/bundle/dms/no-drift/transform.py new file mode 100644 index 00000000000..c07007592a6 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/transform.py @@ -0,0 +1 @@ +# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/no-resources/databricks.yml.tmpl b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl new file mode 100644 index 00000000000..973189b1603 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl @@ -0,0 +1,4 @@ +bundle: + name: dms-no-resources-$UNIQUE_NAME +experimental: + record_deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt new file mode 100644 index 00000000000..a4f993a04e5 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -0,0 +1,80 @@ + +=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-no-resources-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json" +} + +=== Redeploy: the deployment is resolved from that node, so no second deployment is created +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script new file mode 100644 index 00000000000..13924f3664f --- /dev/null +++ b/acceptance/bundle/dms/no-resources/script @@ -0,0 +1,10 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' + +title "Redeploy: the deployment is resolved from that node, so no second deployment is created" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/not-supported/databricks.yml.tmpl b/acceptance/bundle/dms/not-supported/databricks.yml.tmpl new file mode 100644 index 00000000000..9e9352c8506 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-not-supported-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..59b6ebcf2b1 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: the service side is not ready for users yet +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet; remove this setting from your bundle configuration + at experimental.record_deployment_history + in databricks.yml:4:30 + +Name: dms-not-supported-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported-[UNIQUE_NAME]/default + +Found 1 error + +=== The hidden recording variable permits it +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle validate +Name: dms-not-supported-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported-[UNIQUE_NAME]/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..06a237d3445 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,7 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "record_deployment_history is rejected: the service side is not ready for users yet" +trace musterr $CLI bundle validate + +title "The hidden recording variable permits it" +trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..d1c3f78a3ea --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,8 @@ +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +# The parent records via DMS=true, which would permit the field; unset it so the rejection +# path is what runs here. +EnvMatrix.DMS = [""] + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl new file mode 100644 index 00000000000..c46cfb03d73 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-upload-fails-once-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_upload_fails_once_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt b/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml b/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt new file mode 100644 index 00000000000..abdeedeb959 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -0,0 +1,79 @@ + +=== Deploy so there is a recorded resource to recreate +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1 + +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: recording operation for resources.schemas.foo: Fault injected by test. (500 INJECTED) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo?update_mask=state%2Cerror_message%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INJECTED +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script new file mode 100644 index 00000000000..5b609b1b8ee --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy so there is a recorded resource to recreate" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one" +trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/operation-upload-fails-once/test.toml b/acceptance/bundle/dms/operation-upload-fails-once/test.toml new file mode 100644 index 00000000000..5baa9c9d923 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload, once - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl new file mode 100644 index 00000000000..b6aa93265df --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: dms-operation-upload-fails-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five + six: + name: six + seven: + name: seven + eight: + name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt new file mode 100644 index 00000000000..2b273972417 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/output.txt @@ -0,0 +1,4 @@ + +=== An operation upload failure fails the deploy instead of reporting only at the end +>>> grep -c ^Error: LOG.deploy +deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script new file mode 100644 index 00000000000..cc2ead92c6f --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "An operation upload failure fails the deploy instead of reporting only at the end" +# Which resources get refused depends on how far apply got before a background +# upload failed, so the per-resource errors go to a LOG file rather than the diff. +fault.py "PATCH /api/2.0/bundle/*" 500 0 99 INTERNAL_ERROR +errcode $CLI bundle deploy &> LOG.deploy +contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null +trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml new file mode 100644 index 00000000000..b5502a2516c --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -0,0 +1,10 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false + +# Which requests are made depends on how far apply got before a background upload +# failed, so recording them would make the output nondeterministic. +RecordRequests = false + +# Completed versions make DMS the source of truth; unrecorded resources get recreated. +# Deploy must stop rather than continue creating. diff --git a/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl new file mode 100644 index 00000000000..bc5b4ec97ad --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-operation-upload-message-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/operation-upload-message/out.test.toml b/acceptance/bundle/dms/operation-upload-message/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-message/output.txt b/acceptance/bundle/dms/operation-upload-message/output.txt new file mode 100644 index 00000000000..0791010b1f1 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/output.txt @@ -0,0 +1,16 @@ + +=== What a user sees when an operation cannot be recorded +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1 INTERNAL_ERROR + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-operation-upload-message-[UNIQUE_NAME]/default/files... +Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INTERNAL_ERROR) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Fault injected by test. + +Files: 5 uploaded, 0 deleted + +Exit code: 1 diff --git a/acceptance/bundle/dms/operation-upload-message/script b/acceptance/bundle/dms/operation-upload-message/script new file mode 100644 index 00000000000..c6b796e4149 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/script @@ -0,0 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "What a user sees when an operation cannot be recorded" +trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 INTERNAL_ERROR +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/dms/operation-upload-message/test.toml b/acceptance/bundle/dms/operation-upload-message/test.toml new file mode 100644 index 00000000000..1dbd55da1f0 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/test.toml @@ -0,0 +1,8 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false + +# One resource, so which operation is refused is fixed and the error a user sees can be +# asserted. operation-upload-fails covers a deploy of several resources, where which ones +# were reached before the failure landed is not. +RecordRequests = false diff --git a/acceptance/bundle/dms/provenance/databricks.yml.tmpl b/acceptance/bundle/dms/provenance/databricks.yml.tmpl new file mode 100644 index 00000000000..0a3fa1656ae --- /dev/null +++ b/acceptance/bundle/dms/provenance/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-provenance-$UNIQUE_NAME +experimental: + record_deployment_history: true + +targets: + dev: + default: true + mode: development + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt new file mode 100644 index 00000000000..2142dabb700 --- /dev/null +++ b/acceptance/bundle/dms/provenance/output.txt @@ -0,0 +1,68 @@ + +=== Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] +{ + "target_name": "dev", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev" + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script new file mode 100644 index 00000000000..8b15822fa4b --- /dev/null +++ b/acceptance/bundle/dms/provenance/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode" +git-repo-init +git remote add origin https://github.com/databricks/bundle-examples.git +trace $CLI bundle deploy +# The commit SHA changes every run, so assert it is a 40-char hex string and drop it. +add_repl.py "$(git rev-parse HEAD)" COMMIT +trace print_requests.py --dms //versions + +title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' + +# The deploy uploads files in a nondeterministic order; only the requests above are +# asserted. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/test.toml b/acceptance/bundle/dms/provenance/test.toml new file mode 100644 index 00000000000..0a47bfb1b91 --- /dev/null +++ b/acceptance/bundle/dms/provenance/test.toml @@ -0,0 +1,4 @@ +# git-repo-init creates a repo in the test directory so the deploy has git provenance. +Ignore = [ + '.git', +] diff --git a/acceptance/bundle/dms/readplan/databricks.yml.tmpl b/acceptance/bundle/dms/readplan/databricks.yml.tmpl new file mode 100644 index 00000000000..d11621ab1e5 --- /dev/null +++ b/acceptance/bundle/dms/readplan/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: dms-readplan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + # Unique: the Pipelines API refuses a duplicate name across overlapping runs. + name: bar-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + libraries: + - file: + path: ./transform.py diff --git a/acceptance/bundle/dms/readplan/out.test.toml b/acceptance/bundle/dms/readplan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/readplan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/readplan/output.txt b/acceptance/bundle/dms/readplan/output.txt new file mode 100644 index 00000000000..371855dbcc0 --- /dev/null +++ b/acceptance/bundle/dms/readplan/output.txt @@ -0,0 +1,22 @@ + +=== Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan tmp.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created pipelines.bar +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Re-plan: no drift, the deployed job and pipeline carry the deployment stamp +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Both recorded operations carry deployment_id and version_id (lineage), not just kind +>>> print_requests.py --dms //api/2.0/bundle --sort --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.bar", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[UUID]", "sequence_id": "0", "state": "{\"state\":{\"catalog\":\"main\",\"channel\":\"CURRENT\",\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edition\":\"ADVANCED\",\"libraries\":[{\"file\":{\"path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files/transform.py\"}}],\"name\":\"bar-[UNIQUE_NAME]\",\"schema\":\"default\",\"serverless\":true}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-readplan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}, {"resource_key": "pipelines.bar", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/readplan/script b/acceptance/bundle/dms/readplan/script new file mode 100644 index 00000000000..40bd110bf00 --- /dev/null +++ b/acceptance/bundle/dms/readplan/script @@ -0,0 +1,13 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates" +trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json + +title "Re-plan: no drift, the deployed job and pipeline carry the deployment stamp" +trace $CLI bundle plan + +title "Both recorded operations carry deployment_id and version_id (lineage), not just kind" +trace print_requests.py --dms //api/2.0/bundle --sort --oneline | contains.py deployment_id version_id + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/readplan/transform.py b/acceptance/bundle/dms/readplan/transform.py new file mode 100644 index 00000000000..c07007592a6 --- /dev/null +++ b/acceptance/bundle/dms/readplan/transform.py @@ -0,0 +1 @@ +# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl new file mode 100644 index 00000000000..87f0eb4d1a9 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-record-failure-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + doomed: + name: doomed diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt new file mode 100644 index 00000000000..479acca7a84 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -0,0 +1,75 @@ + +=== A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource +>>> fault.py POST /api/2.2/jobs/create 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... +Error: cannot create resources.jobs.doomed: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 6 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-record-failure-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.doomed", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "sequence_id": "0", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{} + +=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script new file mode 100644 index 00000000000..bc0947a1d91 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" +trace fault.py "POST /api/2.2/jobs/create" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/record/databricks.yml.tmpl b/acceptance/bundle/dms/record/databricks.yml.tmpl new file mode 100644 index 00000000000..214793eca38 --- /dev/null +++ b/acceptance/bundle/dms/record/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-record-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/record/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt new file mode 100644 index 00000000000..008731c0126 --- /dev/null +++ b/acceptance/bundle/dms/record/output.txt @@ -0,0 +1,234 @@ + +=== Deploy: the server assigns the deployment ID, and a version + create operation are recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-record-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json" +} + +>>> jq has("deployment_id") .databricks/bundle/default/resources.json +false + +=== Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --get --dms //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: a destroy version and delete operation are recorded, then the deployment is deleted +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py --get --dms //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "2", + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} + +=== Deploy again: the destroy took the node with it, so there is nothing to resolve and a fresh deployment starts at version 1 +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json) doesn't exist. + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json" +} + +>>> print_requests.py --get --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-record-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script new file mode 100644 index 00000000000..ad08fdf1a7f --- /dev/null +++ b/acceptance/bundle/dms/record/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# --get so the reads show up too: this file is the whole DMS call budget, so an extra +# round-trip added to any phase turns up as a diff here. +title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" +trace $CLI bundle deploy +trace print_requests.py --get --dms //api/2.0/bundle + +title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" +# MSYS_NO_PATHCONV prevents Git Bash from rewriting the leading-/ path on Windows. +# Set per command rather than test.toml so trace can export it to its subshell. +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' +trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json + +title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" +rm -rf .databricks +trace $CLI bundle deploy +trace print_requests.py --get --dms //api/2.0/bundle + +title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" +trace $CLI bundle destroy --auto-approve +trace print_requests.py --get --dms //api/2.0/bundle + +title "Deploy again: the destroy took the node with it, so there is nothing to resolve and a fresh deployment starts at version 1" +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" +trace $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' +trace print_requests.py --get --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/resource-lifecycle/databricks.yml.tmpl b/acceptance/bundle/dms/resource-lifecycle/databricks.yml.tmpl new file mode 100644 index 00000000000..808d75dc3be --- /dev/null +++ b/acceptance/bundle/dms/resource-lifecycle/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-resource-lifecycle-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_resource_lifecycle_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/resource-lifecycle/out.requests.txt b/acceptance/bundle/dms/resource-lifecycle/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/resource-lifecycle/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/resource-lifecycle/out.test.toml b/acceptance/bundle/dms/resource-lifecycle/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/resource-lifecycle/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/resource-lifecycle/output.txt b/acceptance/bundle/dms/resource-lifecycle/output.txt new file mode 100644 index 00000000000..475995d9c71 --- /dev/null +++ b/acceptance/bundle/dms/resource-lifecycle/output.txt @@ -0,0 +1,180 @@ + +=== Deploy: the state write records the resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-resource-lifecycle-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_resource_lifecycle_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_resource_lifecycle_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Recreate writes state twice - the entry is dropped, then the new resource is saved +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Recreated schemas.foo +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "dms_other_[UNIQUE_NAME].dms_resource_lifecycle_schema_[UNIQUE_NAME]", + "sequence_id": "1", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_resource_lifecycle_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: the delete is recorded with the id and no state +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "2", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/resource-lifecycle/script b/acceptance/bundle/dms/resource-lifecycle/script new file mode 100644 index 00000000000..43050cc7c8f --- /dev/null +++ b/acceptance/bundle/dms/resource-lifecycle/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the state write records the resource" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Recreate writes state twice - the entry is dropped, then the new resource is saved" +# One operation per resource per version; both writes land on the same one. +# The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED. +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle + +title "Destroy: the delete is recorded with the id and no state" +trace $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl new file mode 100644 index 00000000000..01d891c3880 --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-serialized-plan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/serialized-plan/out.test.toml b/acceptance/bundle/dms/serialized-plan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/serialized-plan/output.txt b/acceptance/bundle/dms/serialized-plan/output.txt new file mode 100644 index 00000000000..fc84deb9f4c --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/output.txt @@ -0,0 +1,41 @@ + +=== Deploy once to create the deployment (records version 1) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) +>>> [CLI] bundle plan -o json + +>>> jq {deployment_id, last_version_id, next_version_id} tmp.plan.json +{ + "deployment_id": "[NUMID]", + "last_version_id": "1", + "next_version_id": "2" +} + +=== Change the job and deploy again: advances the deployment to version 2 +>>> update_file.py databricks.yml name: foo name: foo-v2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== Replaying the earlier plan is rejected: it predates the deployment's current version +>>> musterr [CLI] bundle deploy --plan tmp.plan.json +Error: this plan predates the deployment's current version 2; run 'bundle plan' again + + +=== Only versions 1 and 2 were recorded; the rejected replay created none +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-serialized-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo-v2\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/serialized-plan/script b/acceptance/bundle/dms/serialized-plan/script new file mode 100644 index 00000000000..0800e1a8265 --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy once to create the deployment (records version 1)" +trace $CLI bundle deploy + +title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" +trace $CLI bundle plan -o json > tmp.plan.json +trace jq '{deployment_id, last_version_id, next_version_id}' tmp.plan.json + +title "Change the job and deploy again: advances the deployment to version 2" +trace update_file.py databricks.yml "name: foo" "name: foo-v2" +trace $CLI bundle deploy + +title "Replaying the earlier plan is rejected: it predates the deployment's current version" +trace musterr $CLI bundle deploy --plan tmp.plan.json + +title "Only versions 1 and 2 were recorded; the rejected replay created none" +trace print_requests.py --dms //api/2.0/bundle --oneline + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl new file mode 100644 index 00000000000..dea8fbd9a8d --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-stale-plan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/stale-plan/out.test.toml b/acceptance/bundle/dms/stale-plan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt new file mode 100644 index 00000000000..2ce3b598d1c --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -0,0 +1,21 @@ + +=== Save a plan, then deploy it: records version 1 +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan tmp.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Replay: re-deploying the same plan is rejected - it predates the recorded version +>>> musterr [CLI] bundle deploy --plan tmp.plan.json +Error: this plan predates the deployment's current version 1; run 'bundle plan' again + + +=== Only the first deploy recorded a version; the rejected replay created none +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-stale-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script new file mode 100644 index 00000000000..1f9e792f30f --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/script @@ -0,0 +1,13 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Save a plan, then deploy it: records version 1" +trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json + +title "Replay: re-deploying the same plan is rejected - it predates the recorded version" +trace musterr $CLI bundle deploy --plan tmp.plan.json + +title "Only the first deploy recorded a version; the rejected replay created none" +trace print_requests.py --dms //api/2.0/bundle --oneline + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/summary/databricks.yml.tmpl b/acceptance/bundle/dms/summary/databricks.yml.tmpl new file mode 100644 index 00000000000..1c2fa3c7f20 --- /dev/null +++ b/acceptance/bundle/dms/summary/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-summary-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt new file mode 100644 index 00000000000..233f2416dc3 --- /dev/null +++ b/acceptance/bundle/dms/summary/output.txt @@ -0,0 +1,37 @@ + +=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + +=== Redeploying advances the version the summary reports +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + +=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script new file mode 100644 index 00000000000..75ec8612ffe --- /dev/null +++ b/acceptance/bundle/dms/summary/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "Redeploying advances the version the summary reports" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" +trace $CLI bundle destroy --auto-approve +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' + +# This test asserts the summary output, not the requests behind it; the file uploads +# recorded here are ordered nondeterministically. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml new file mode 100644 index 00000000000..32a52823ff7 --- /dev/null +++ b/acceptance/bundle/dms/test.toml @@ -0,0 +1,19 @@ +Cloud = true + +# Deployment Metadata Service (DMS) recording is only supported by the direct +# engine; it is a no-op on terraform. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + + +RecordRequests = true + +Ignore = [ + '.databricks', + 'databricks.yml', +] + +# These tests set experimental.record_deployment_history in databricks.yml; the DMS env var +# both records and permits that otherwise-gated field. bundle/dms/not-supported covers the +# rejection path. Pinned to "true" (not the suite's ["", "true"]) since the field already +# records - the empty variant would just duplicate it. +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml new file mode 100644 index 00000000000..a7077f18b1f --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-version-never-created + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt new file mode 100644 index 00000000000..beadce201e3 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -0,0 +1,36 @@ + +=== The first version fails, so no deployment record exists - only the node naming its ID +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + +Files: 5 uploaded, 0 deleted + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT" +} + +=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + +Files: 2 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script new file mode 100644 index 00000000000..462e18de914 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/script @@ -0,0 +1,7 @@ +title "The first version fails, so no deployment record exists - only the node naming its ID" +trace musterr $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' + +title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" +trace musterr $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml new file mode 100644 index 00000000000..6553e7c09bb --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -0,0 +1,12 @@ +# Local only: the failure has to land on CreateVersion and not on the CreateDeployment +# that precedes it, which fault.py's trailing-* pattern cannot express, so this case +# keeps a [[Server]] stub - and the cloud harness ignores stubs. +Cloud = false + +# The first version fails, so the deployment record is never created - only the +# workspace node CreateDeployment registered. The next deploy resolves the ID from +# that node and has to cope with a deployment that has no record yet. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index a03c479c071..01ed00ce42f 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -8,7 +8,7 @@ # Resolved config the engines see. Today every "" field is still present here; a # fix in the initialize phase would drop them, and this golden would show that. -$CLI bundle validate -o json -t direct | nostamp | jq .resources > out.validate.json +$CLI bundle validate -o json -t direct | nostamp.py | jq .resources > out.validate.json # Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the # deployment history calls the DMS run adds. diff --git a/acceptance/bundle/environments/dependencies/script b/acceptance/bundle/environments/dependencies/script index 00e5fc846cc..77c215da892 100644 --- a/acceptance/bundle/environments/dependencies/script +++ b/acceptance/bundle/environments/dependencies/script @@ -4,11 +4,11 @@ set -euo pipefail trace $CLI bundle validate trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp.py -trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp.py -trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs.test_job.environments' -trace $CLI bundle validate -o json | nostamp | jq '.resources.pipelines.test_pipeline.environment' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs.test_job.environments' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.pipelines.test_pipeline.environment' rm out.requests.txt diff --git a/acceptance/bundle/escaped_refs/out.plan.direct.json b/acceptance/bundle/escaped_refs/out.plan.direct.json index ae4fd743d26..a64f09b477b 100644 --- a/acceptance/bundle/escaped_refs/out.plan.direct.json +++ b/acceptance/bundle/escaped_refs/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.example_ingestion_job": { diff --git a/acceptance/bundle/escaped_refs/out.test.toml b/acceptance/bundle/escaped_refs/out.test.toml index bf4f0e7d3c6..d5b9a6f4a16 100644 --- a/acceptance/bundle/escaped_refs/out.test.toml +++ b/acceptance/bundle/escaped_refs/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/escaped_refs/output.txt b/acceptance/bundle/escaped_refs/output.txt index 6e12dbfd8fc..342535d0037 100644 --- a/acceptance/bundle/escaped_refs/output.txt +++ b/acceptance/bundle/escaped_refs/output.txt @@ -15,7 +15,7 @@ Created jobs.example_ingestion_job Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/escaped_refs/script b/acceptance/bundle/escaped_refs/script index 34640492656..1b8986539c6 100644 --- a/acceptance/bundle/escaped_refs/script +++ b/acceptance/bundle/escaped_refs/script @@ -12,10 +12,12 @@ title "Validate: escapes are still escaped in the config" trace $CLI bundle validate -o json | jq '.resources.jobs.example_ingestion_job.tasks[0].notebook_task.base_parameters' title "Plan (json)" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. title "Deploy: payload sent to the jobs API\n" $CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) -trace print_requests.py //jobs +# --nostamp: under deployment-history recording the job payload carries the DMS deployment +# stamp; drop it so the assertion is the same whether or not recording is on. +trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/escaped_refs/test.toml b/acceptance/bundle/escaped_refs/test.toml index e7b9cac3bf1..94b36638c0e 100644 --- a/acceptance/bundle/escaped_refs/test.toml +++ b/acceptance/bundle/escaped_refs/test.toml @@ -8,3 +8,7 @@ Ignore = [ ".databricks", "databricks.yml", ] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/dashboard-inplace/test.toml b/acceptance/bundle/generate/dashboard-inplace/test.toml index d5511c15564..10eef9f96c4 100644 --- a/acceptance/bundle/generate/dashboard-inplace/test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/test.toml @@ -1,3 +1,7 @@ +# bundle generate reading recorded state from DMS is a fast followup; until then this +# deploy-then-generate flow can't run under recording (local state is a tombstone). +EnvMatrix.DMS = [""] + [[Repls]] Old = "[0-9a-f]{32}" New = "[DASHBOARD_ID]" diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/generate/genie_space_inplace/out.test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/genie_space_inplace/test.toml b/acceptance/bundle/generate/genie_space_inplace/test.toml index 723c9907976..91944fc581e 100644 --- a/acceptance/bundle/generate/genie_space_inplace/test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/test.toml @@ -1,5 +1,8 @@ # Genie spaces are only deployed via the direct deployment engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# bundle generate reading recorded state from DMS is a fast followup; until then this +# deploy-then-generate flow can't run under recording (local state is a tombstone). +EnvMatrix.DMS = [""] [[Repls]] Old = "[0-9a-f]{32}" diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json b/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json index cbaa60b7ace..e564ae92773 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/auto-migrate-clean/script b/acceptance/bundle/migrate/auto-migrate-clean/script index 0f2eb7c1ce1..5afad5a39c5 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/script +++ b/acceptance/bundle/migrate/auto-migrate-clean/script @@ -31,7 +31,7 @@ rm -f out.requests.txt title "Modify the config — plan shows the pending update" trace update_file.py databricks.yml "Test Auto-Migrate Job" "Test Auto-Migrate Job (renamed)" trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.direct.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.direct.json title "Deploy picks up the update (with and without --plan)\n" $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/migrate/auto-migrate-clean/test.toml b/acceptance/bundle/migrate/auto-migrate-clean/test.toml index ff8a66c196e..32db4804fbf 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/test.toml @@ -1 +1,5 @@ EnvMatrix.READPLAN = ["", "1"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/script b/acceptance/bundle/migrate/auto-migrate-direct-only/script index 3cb22cb053a..7f9c18cac46 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only/script +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml "name: test-bundle" $'name: test-bundle\n e title "Plan skips the direct-only resource, since this run still uses terraform" trace $CLI bundle plan -trace $CLI bundle plan -o json | gron.py | grep instance_pools +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep instance_pools title "Deploy migrates the state, skipping the direct-only resource" trace $CLI bundle deploy diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index 7b9967a085d..fec804b95b3 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/basic/script b/acceptance/bundle/migrate/basic/script index 3247e9fad8d..64c42e6d641 100755 --- a/acceptance/bundle/migrate/basic/script +++ b/acceptance/bundle/migrate/basic/script @@ -43,7 +43,7 @@ rm out.requests.txt title "Update databricks.yml, run plan & update" trace update_file.py databricks.yml Migration Migrated trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_update.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy title "Different target, still on terraform" diff --git a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json index cbdc8b8203b..66f20559a9a 100644 --- a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json +++ b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/dashboards/script b/acceptance/bundle/migrate/dashboards/script index bd599144f94..b2c9639a0fd 100755 --- a/acceptance/bundle/migrate/dashboards/script +++ b/acceptance/bundle/migrate/dashboards/script @@ -15,7 +15,7 @@ rm out.requests.txt # Badness: known issue with dashboards, will show up as 'recreate' see https://github.com/databricks/cli/pull/3966 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 unchanged" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan | contains.py "1 unchanged" -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_after_migrate.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_after_migrate.json trace print_requests.py --get //dashboards | contains.py 'engine/direct' > /dev/null trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy --auto-approve @@ -32,7 +32,7 @@ rm out.requests.txt #title "Update databricks.yml, run plan & update" #trace update_file.py databricks.yml dashboard DASHBOARD #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_update.json +#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy #rm out.requests.txt diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json index 3e01413959f..84de0f4ffdc 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json index e21c3b33069..a54d3357cb1 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/default-python/script index e3a0b9b3e05..6e05ab85f7f 100755 --- a/acceptance/bundle/migrate/default-python/script +++ b/acceptance/bundle/migrate/default-python/script @@ -11,7 +11,7 @@ trace print_state.py > ../out.state_after_migration.json trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_migration.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_migration.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl and num_workers trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy @@ -21,5 +21,5 @@ rm ../out.state_after_deploy.json title "Extra plan: should have no drift" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_deploy.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_deploy.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl diff --git a/acceptance/bundle/migrate/runas/out.plan.json b/acceptance/bundle/migrate/runas/out.plan.json index 0888cdc22ef..56992a01bc0 100644 --- a/acceptance/bundle/migrate/runas/out.plan.json +++ b/acceptance/bundle/migrate/runas/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/runas/script index f3675f4d9bb..4affba0a373 100644 --- a/acceptance/bundle/migrate/runas/script +++ b/acceptance/bundle/migrate/runas/script @@ -23,4 +23,4 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate trace print_state.py > out.new_state.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan.json diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 2d7df920e29..5370251a14d 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -18,7 +18,7 @@ Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs --sort +>>> print_requests.py //jobs --sort --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index 413394d8b7d..c53b9b85671 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -12,4 +12,6 @@ trace jq '.plan | map_values(.depends_on)' plan.json # that the payload below is identical either way. title "Payload: escaped stays literal, real is resolved\n" $CLI bundle deploy $(readplanarg plan.json) -trace print_requests.py //jobs --sort +# --nostamp: under deployment-history recording the job payload carries the DMS deployment +# stamp; drop it so the assertion is the same whether or not recording is on. +trace print_requests.py //jobs --sort --nostamp diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index 3bb4c8310c8..424b73dc67b 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.schema_a": { diff --git a/acceptance/bundle/resource_deps/grant_ref/script b/acceptance/bundle/resource_deps/grant_ref/script index 82800b1654e..ad897f76987 100644 --- a/acceptance/bundle/resource_deps/grant_ref/script +++ b/acceptance/bundle/resource_deps/grant_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json index f004dcdfcde..1c0e5115a32 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.a": { diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json index fa72eb2700b..343ca061a3b 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json index 0c9904aec79..192e2dd8c2b 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/id_chain/script b/acceptance/bundle/resource_deps/id_chain/script index bcd103ac2a5..eada43ba1c9 100644 --- a/acceptance/bundle/resource_deps/id_chain/script +++ b/acceptance/bundle/resource_deps/id_chain/script @@ -16,7 +16,7 @@ print_requests_short() { trace $CLI bundle plan trace print_requests -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle deploy @@ -25,7 +25,7 @@ trace print_requests_short trace update_file.py databricks.yml aa_desc aa_new_desc trace update_file.py databricks.yml prefix new_prefix -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle plan @@ -34,5 +34,5 @@ trace print_requests trace $CLI bundle deploy trace print_requests_short -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json b/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json index 7c32ae3fe4c..b955e17cf46 100644 --- a/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.a": { diff --git a/acceptance/bundle/resource_deps/id_star/script b/acceptance/bundle/resource_deps/id_star/script index 6b67f59b7b8..46366e7706a 100644 --- a/acceptance/bundle/resource_deps/id_star/script +++ b/acceptance/bundle/resource_deps/id_star/script @@ -1 +1 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json index 43c1928ebb8..4cdbdd7e630 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index 1bfbc8fe053..c6a1b111aca 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id/script b/acceptance/bundle/resource_deps/job_id/script index f8d30917d1c..96d8aed011f 100644 --- a/acceptance/bundle/resource_deps/job_id/script +++ b/acceptance/bundle/resource_deps/job_id/script @@ -1,6 +1,6 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -8,7 +8,7 @@ foo_id=`read_id.py foo` bar_id=`read_id.py bar` cp empty.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index 64087a8c63c..8aa9e5e2544 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script index 03b9a087810..8fd0af69437 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/script +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete bar, keep foo (foo config updated to not depend on bar)" cp only_foo.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy # Sort requests because foo's update and bar's delete have no dependency edge # (foo's dependency on bar was removed in only_foo.yml), so execution order is non-deterministic. diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json index 6c6c9577e8f..83132ceb0b4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/script b/acceptance/bundle/resource_deps/job_id_delete_foo/script index f5efbd22609..30e1194c4cd 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/script +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete foo, keep bar" cp only_bar.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/jobs_update/script b/acceptance/bundle/resource_deps/jobs_update/script index f806f2db846..b1fc7332355 100644 --- a/acceptance/bundle/resource_deps/jobs_update/script +++ b/acceptance/bundle/resource_deps/jobs_update/script @@ -21,8 +21,8 @@ trace $CLI bundle plan title "Fetch job ID and verify remote state" # Badness: output should not be different per engine; investigate if it's the same on cloud -trace $CLI jobs get $foo_id | nostamp > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json -trace $CLI jobs get $bar_id | nostamp +trace $CLI jobs get $foo_id | nostamp.py > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI jobs get $bar_id | nostamp.py rm out.requests.txt trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json index d768b766d36..be2fb1b17e7 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json index 77e77c19eb1..deb96f123e6 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index d456ed8b023..89a3fc4c36f 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -1,5 +1,5 @@ echo "*" > .gitignore -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -13,7 +13,7 @@ bar_id=`read_id.py bar` title "Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged" trace envsubst < job_update.json > tmp.json && mv tmp.json job_update.json trace $CLI jobs reset --json @job_update.json -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/missing_string_field/script b/acceptance/bundle/resource_deps/missing_string_field/script index c526e980444..d96e2f30b0a 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/script +++ b/acceptance/bundle/resource_deps/missing_string_field/script @@ -1,4 +1,4 @@ -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources errcode $CLI bundle plan trace print_requests.py --nostamp //pipeline diff --git a/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json index d7c0cc7b12f..8dea931a43e 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_a": { diff --git a/acceptance/bundle/resource_deps/permission_ref/script b/acceptance/bundle/resource_deps/permission_ref/script index 82800b1654e..ad897f76987 100644 --- a/acceptance/bundle/resource_deps/permission_ref/script +++ b/acceptance/bundle/resource_deps/permission_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json index 84d1505045e..60da159159b 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json index 7b3f986f02c..34c1baf3067 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json index 4f7329005fb..222f87d5afb 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index 691e24279b2..b1a6741f081 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -1,7 +1,7 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.create.requests.json @@ -14,7 +14,7 @@ trace $CLI bundle plan # empty title "Update storage, triggering recreate for pipeline; this means updating downstream deps" trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/my-new-storage" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -23,13 +23,13 @@ foo_id_2=`read_id.py foo` title "Fetch resource IDs and verify remote state" trace musterr $CLI pipelines get $foo_id -trace $CLI pipelines get $foo_id_2 | nostamp -trace $CLI jobs get $bar_id | nostamp| jq 'del(.settings.run_as)' +trace $CLI pipelines get $foo_id_2 | nostamp.py +trace $CLI jobs get $bar_id | nostamp.py| jq 'del(.settings.run_as)' rm out.requests.txt title "Follow up plan & deploy do nothing" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json index ef8c827e130..869b15ad6f7 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo1": { diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json index 0bb3be878f6..ac8af43c069 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/remote_pipeline/script b/acceptance/bundle/resource_deps/remote_pipeline/script index b8683429aac..4df5a844e73 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/script +++ b/acceptance/bundle/resource_deps/remote_pipeline/script @@ -3,8 +3,8 @@ print_requests() { rm out.requests.txt } -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json b/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json index bfa6735a725..6f4ff29a254 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.mypipeline": { diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script index cc0db790b5c..c04b243240b 100644 --- a/acceptance/bundle/resource_deps/resources_var/script +++ b/acceptance/bundle/resource_deps/resources_var/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy -t dev &> out.deploy.txt trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt trace print_telemetry_bool_values | grep -v direct_drymigrate diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json index 748830efbe4..cb7acf0c2d8 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.mypipeline": { diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script index 707ca1f347f..2fb906b4945 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json -t dev > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json b/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json index 37b90504345..fed77573273 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.dst_git": { diff --git a/acceptance/bundle/resource_deps/tf_path_renames/script b/acceptance/bundle/resource_deps/tf_path_renames/script index 568894d3ba9..dc491ff0f5e 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/script +++ b/acceptance/bundle/resource_deps/tf_path_renames/script @@ -1,3 +1,3 @@ -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs | jq -rs '[.[].body.name] | sort[]' diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json index 1fecd0a7b5a..081c3abd05d 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.volumes.bar": { diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/script b/acceptance/bundle/resource_deps/volume_path_contains_id/script index 914f75443d2..21d91ccd863 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/script +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/script @@ -5,7 +5,7 @@ cleanup() { trace $CLI bundle validate -o json | jq .resources trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trap cleanup EXIT trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json index a6cab2ee51e..9bca8261612 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index 6c7250cc564..21a045fa49c 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,5 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs, .resources.volumes' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs, .resources.volumes' # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml index b53e5e7fbe8..10ce9d961a6 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -6,3 +6,7 @@ RecordRequests = true # so the case is unsupported there (see CHANGELOG.md). Direct is the # default engine, so the motivating use case works out of the box. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/script b/acceptance/bundle/resources/apps/config-drift-stopped/script index c449a3099a3..50e046d9442 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/script +++ b/acceptance/bundle/resources/apps/config-drift-stopped/script @@ -10,10 +10,10 @@ trace $CLI bundle deploy title "Change config while running: drift is detected (active deployment present)" trace update_file.py databricks.yml original_value changed_value -trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' title "Stop the app: the backend clears the active deployment" trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' title "Same config change is now skipped: no active deployment to compare against" -trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' diff --git a/acceptance/bundle/resources/apps/config-drift/script b/acceptance/bundle/resources/apps/config-drift/script index 4728255dc8f..23154a3fdf9 100644 --- a/acceptance/bundle/resources/apps/config-drift/script +++ b/acceptance/bundle/resources/apps/config-drift/script @@ -9,7 +9,7 @@ trap cleanup EXIT trace $CLI bundle deploy title "Verify no drift after deploy" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json SOURCE_CODE_PATH=$(trace $CLI apps get $UNIQUE_NAME --output json | jq -r '.active_deployment.source_code_path') @@ -27,13 +27,13 @@ $CLI apps deploy $UNIQUE_NAME --no-wait --json '{ title "Plan should detect config drift" trace $CLI bundle plan # Skip entries with action "skip" -$CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json title "Redeploy to fix drift" trace $CLI bundle deploy title "Verify no drift after fix" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json # TODO: add test for git_source drift when git_source is supported in the Deploy API # Currently it fails with the error: Git source reference is required diff --git a/acceptance/bundle/resources/apps/config-no-deployment/script b/acceptance/bundle/resources/apps/config-no-deployment/script index f1970999645..3b7e38e3493 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/script +++ b/acceptance/bundle/resources/apps/config-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: config/source_code_path drift is skipped while the app h trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "Both deploy-only fields are skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | jq '.plan[].changes | {config, source_code_path}' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {config, source_code_path}' diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/script b/acceptance/bundle/resources/apps/git-source-no-deployment/script index 63cce6312c3..28e12809b2e 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/script +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: git_source drift is skipped while the app has no active trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "git_source is skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | jq '.plan[].changes | {git_source}' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {git_source}' diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index dd08f33fff5..8bdbb4dfd9c 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json | nostamp > LOG.planjson +$CLI bundle plan -o json | nostamp.py > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/script b/acceptance/bundle/resources/apps/uc-securable-drift/script index 04438951475..7f3022a2bb1 100644 --- a/acceptance/bundle/resources/apps/uc-securable-drift/script +++ b/acceptance/bundle/resources/apps/uc-securable-drift/script @@ -10,4 +10,4 @@ title "Re-plan is clean: the TABLE securable's server-computed securable_kind is trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "securable_kind is classified output-only (skip); the VOLUME securable has none" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/script b/acceptance/bundle/resources/catalogs/comment_out_of_band/script index fe406882746..c0f21c30082 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/catalogs/$CATALOG" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the catalog" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/script b/acceptance/bundle/resources/catalogs/drift/managed_properties/script index 9413632bb9c..25664d4eefd 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/script +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateCatalog call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index c7af4b9a333..cdf7731c99b 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.test_cluster": { @@ -17,7 +17,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -97,7 +97,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, @@ -180,7 +180,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, @@ -265,7 +265,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 71d7b6860af..00614a0ec10 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -10,11 +10,11 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist with num_workers after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' @@ -26,7 +26,7 @@ update_file.py databricks.yml " num_workers: 2" " autoscale: min_workers: 2 max_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -38,7 +38,7 @@ title "Changing autoscale should call update API on stopped cluster\n" update_file.py databricks.yml "min_workers: 2" "min_workers: 3" update_file.py databricks.yml "max_workers: 4" "max_workers: 5" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -53,7 +53,7 @@ title "Changing autoscale should call resize API on running cluster\n" update_file.py databricks.yml "min_workers: 3" "min_workers: 4" update_file.py databricks.yml "max_workers: 5" "max_workers: 6" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -66,7 +66,7 @@ update_file.py databricks.yml " autoscale: min_workers: 4 max_workers: 6" " num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json index b4ba1c76152..9d3e514ecfe 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.test_cluster": { @@ -20,7 +20,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -92,7 +92,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, @@ -164,7 +164,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 14799220348..7643752ef88 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -10,18 +10,18 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' title "Changing num_workers should call update API on stopped cluster\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -35,7 +35,7 @@ $CLI clusters start "${CLUSTER_ID}" title "Changing num_workers should call resize API on running cluster\n" update_file.py databricks.yml "num_workers: 3" "num_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -47,7 +47,7 @@ title "Changing num_workers and spark_conf should call update API\n" update_file.py databricks.yml "num_workers: 4" "num_workers: 5" update_file.py databricks.yml '"spark.executor.memory": "2g"' '"spark.executor.memory": "4g"' $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json index e14ec2fba2a..8b496224317 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml index bf4f0e7d3c6..d5b9a6f4a16 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/script b/acceptance/bundle/resources/dashboards/change-serialized-inline/script index 03b8c2ac46b..41629784e3f 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/script @@ -23,7 +23,7 @@ update_file.py databricks.yml "Page 1" "Page One" trace $CLI bundle plan | contains.py "update" "!recreate" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "deploy the change\n" $CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml index 5cab5faff69..fae8d4e5d0e 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml @@ -8,3 +8,7 @@ Ignore = [ ] EnvMatrix.READPLAN = ["", "1"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json index 01e4ab17548..7686609f862 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script index aa3aa4026b7..e5a1a51185a 100755 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script @@ -29,7 +29,7 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path}' echo "$DASHBOARD" | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}' # Verify that there is no drift right after deploy. -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Modify the direct plan to replace "serialized_dashboard" with a fixture. # It is normalized on the backend so we cannot compare reliably across local and cloud. diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json index 6cedd195b0f..c4d8bc38fd9 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script index 2c1396ac618..b8928da1198 100755 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script @@ -25,7 +25,7 @@ trace $CLI lakeview trash $DASHBOARD_ID trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Run destroy - should succeed even though dashboard is already trashed. # -qq drops the summary: terraform refreshes the trashed dashboard away and reports diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json index c177eb0b450..e11fc5079b5 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index 2b4966fd87d..d7c2d18037b 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -38,7 +38,7 @@ add_repl "$(echo "$UPDATE_RESP" | jq -r '.etag')" ETAG_2 title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy title "Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored:" diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index aad5ccf5498..bd29c0328af 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml index 96b263c004a..ca3a5884557 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml @@ -3,3 +3,6 @@ RecordRequests = false RunsOnDbr = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + +# bundle generate reading recorded state from DMS is a fast followup; opt out of recording. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script index 41022465670..4c455c0c3f5 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script @@ -16,7 +16,7 @@ ETAG_1=$(echo "$DASHBOARD" | jq -r '.etag') add_repl "$ETAG_1" ETAG_1 echo "$DASHBOARD" | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" rm out.requests.txt # Inject a single publish failure for the update below. @@ -34,7 +34,7 @@ ETAG_2=$(retry --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.e add_repl "$ETAG_2" ETAG_2 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" # Bug: re-running deploy fails with "modified remotely" because the stored etag # (pre-PATCH) no longer matches the remote etag (bumped by the PATCH above). diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script index 9cd9a12869f..cd278bab258 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script @@ -16,7 +16,7 @@ DASHBOARD_ID=$(read_id.py dashboard1) # A clean re-plan is a no-op: the published content matches the draft. title "Plan right after deploy -- published is current, no change:" -trace $CLI bundle plan -o json | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' # Make an out-of-band DRAFT update without republishing. This advances the draft's # update_time past the published revision_create_time, so the published content is now @@ -28,4 +28,4 @@ DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEF $CLI lakeview update "$DASHBOARD_ID" --json "${DASHBOARD_JSON}" > /dev/null title "Plan after out-of-band draft update -- a republish is owed:" -trace $CLI bundle plan -o json | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' diff --git a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json index 5e6ad9eb0a6..ad1171713f2 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/simple/script b/acceptance/bundle/resources/dashboards/simple/script index 4c8b9b55ace..6e2b14529f1 100644 --- a/acceptance/bundle/resources/dashboards/simple/script +++ b/acceptance/bundle/resources/dashboards/simple/script @@ -24,4 +24,4 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path, serialized_dashboar # Verify that there is no drift right after deploy. We use |= to update the serialized_dashboard field in place to account # for formatting that the cloud server applies to the serialized_dashboard field. -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json index 852cc492879..cd96018988d 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json index b3efa15d5da..42a8d2983bf 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.dashboards.dashboard1": { diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index c3af11af2f7..ddb2b9e8b34 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json # Deploy the dashboard trace $CLI bundle deploy -qq @@ -25,7 +25,7 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -qq diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 02b174d3869..60aedcde18e 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script index 812cfb9abd4..9e5892a1416 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/script +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -15,4 +15,4 @@ add_repl "$GENIE_SPACE_ID" GENIE_SPACE_ID # Without normalization the inline serialized_space leaves a map in the # config struct while state holds a string, and structdiff reports false # drift on every plan. -trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/script b/acceptance/bundle/resources/genie_spaces/version_migration/script index 141efd5bef6..5615bbb7034 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/script +++ b/acceptance/bundle/resources/genie_spaces/version_migration/script @@ -19,7 +19,7 @@ trace $CLI bundle plan # (and the etag guarding it) is version 2. The serialized_space change is # recorded but skipped via the etag_based rule. title "serialized_space: local version 1, remote version 2, skipped (etag_based)" -$CLI bundle plan -o json \ +$CLI bundle plan -o json | nostamp.py \ | jq '.plan["resources.genie_spaces.foo"].changes.serialized_space | {action, reason, local_version: (.new | fromjson | .version), diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 51f6d220e56..008571de908 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.catalogs.grants_catalog": { diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index fa674375af2..afefaf2f192 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/catalogs/script b/acceptance/bundle/resources/grants/catalogs/script index 067710b705e..aa47ee0ce94 100755 --- a/acceptance/bundle/resources/grants/catalogs/script +++ b/acceptance/bundle/resources/grants/catalogs/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml CREATE_SCHEMA USE_SCHEMA -trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json index ab6c291b239..66689f09124 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.registered_models.my_registered_model": { diff --git a/acceptance/bundle/resources/grants/registered_models/script b/acceptance/bundle/resources/grants/registered_models/script index 98d73773afc..43ec2d3b87a 100644 --- a/acceptance/bundle/resources/grants/registered_models/script +++ b/acceptance/bundle/resources/grants/registered_models/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json index 534d0328aa1..429898e1b67 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.grants_schema": { diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index d8bbf97ffd6..46444558c43 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/script b/acceptance/bundle/resources/grants/schemas/change_privilege/script index 737f7491d3a..ec907ef8887 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/script +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml USE_SCHEMA APPLY_TAG -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json index ff320b18213..f05a65c1a5d 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script index 12a255773b1..1056a114bfb 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script @@ -11,4 +11,4 @@ trace $CLI bundle deploy print_requests.py --get //permissions --keep > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt # effective_predictive_optimization_flag is inherited from the metastore and backend-controlled; drop it so the test does not depend on metastore settings. # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json index cc1ae0e9cf3..bc969eb2f1c 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json index bf38acd93fe..8021983f5b5 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script index 6f8965cfe1a..fe3b395156d 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script @@ -12,7 +12,7 @@ trap cleanup EXIT trace $CLI bundle deploy trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # remote_state.__embed__ order is non-deterministic; extract and sort separately into per-engine file jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl > out.embed.$DATABRICKS_BUNDLE_ENGINE.txt tmp=$(mktemp) @@ -21,4 +21,4 @@ tmp=$(mktemp) # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index f49832eddc9..4e059e5634b 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.grants_schema": { diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index 1daacee25b4..9042bde70e8 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/volumes/script b/acceptance/bundle/resources/grants/volumes/script index 725049cda8e..9b8854c8062 100644 --- a/acceptance/bundle/resources/grants/volumes/script +++ b/acceptance/bundle/resources/grants/volumes/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -10,7 +10,7 @@ trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq -- update_file.py databricks.yml WRITE_VOLUME MANAGE # different on cloud vs local due to remote state -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 0fe24556182..2c1ee0fa812 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -15,7 +15,7 @@ musterr trace $CLI bundle deploy # The framework saves the run id before WaitAfterCreate, so FAILED vs SUCCESS is drift. title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run -trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' # Name the job too, so the run-now bodies below say which number it is. read_id.py my_job > /dev/null diff --git a/acceptance/bundle/resources/job_runs/redeploy/script b/acceptance/bundle/resources/job_runs/redeploy/script index 8ceb3a88052..2c969f0a969 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/script +++ b/acceptance/bundle/resources/job_runs/redeploy/script @@ -17,7 +17,7 @@ trace print_requests.py //jobs/run-now title "change the run configuration and redeploy" trace update_file.py databricks.yml "env: dev" "env: prod" -trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' trace $CLI bundle deploy read_id.py my_run > /dev/null trace $CLI bundle summary diff --git a/acceptance/bundle/resources/job_runs/wait/script b/acceptance/bundle/resources/job_runs/wait/script index 70cee1c9082..3a68150df3b 100644 --- a/acceptance/bundle/resources/job_runs/wait/script +++ b/acceptance/bundle/resources/job_runs/wait/script @@ -15,5 +15,5 @@ downstream_id=$(read_id.py downstream_job) trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" title "nothing the run resolved is drift, so a redeploy starts no new run" -trace $CLI bundle plan -o json | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' +trace $CLI bundle plan -o json | nostamp.py | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json index 6328c4f6dd7..aaf9e6c7458 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index b5823022572..00188cda82d 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) trace print_requests.py //jobs --nostamp print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index b15d8f9e757..8957ed9ce06 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -3,8 +3,8 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] [[Repls]] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json b/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json index f5f1717411e..126e0cbda1b 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index 1622b3e11cb..f1bf01f596d 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json index a881f097c7a..0539fa4c2de 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json index 7a64f0a18d6..7e37e4e09e1 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/delete_task/script b/acceptance/bundle/resources/jobs/delete_task/script index 29b856e5ceb..89cf79be7e3 100644 --- a/acceptance/bundle/resources/jobs/delete_task/script +++ b/acceptance/bundle/resources/jobs/delete_task/script @@ -1,6 +1,6 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) grep -v TO_DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index 46a4d91b848..04d6ea1e5b3 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,5 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script index 6f0606bc346..55c9b9bbf82 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json index 693492859fc..052acba00b3 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index 3a651737979..b2b2fefd0ba 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json index c9493e8a21c..796dbf0528b 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index 46a4d91b848..04d6ea1e5b3 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,5 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json index 3af240a0ed0..94e8d4554e1 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 9ca6a0655b6..b6ed653c0ef 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,7 +13,7 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/script b/acceptance/bundle/resources/jobs/tags_empty_map/script index 4a232723bd0..d3dd04214cf 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/script +++ b/acceptance/bundle/resources/jobs/tags_empty_map/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json index e13d1b307a5..4a7b31e1946 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json index 7aecad7102e..87975ef1bc0 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json index 56eb1211e99..e81f442dbab 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index 1be869aa509..7a90cf02255 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -1,12 +1,12 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) trace print_requests.py //jobs --nostamp > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) trace print_requests.py //jobs --nostamp @@ -14,7 +14,7 @@ trace print_requests.py //jobs --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) trace print_requests.py //jobs --nostamp | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index 20c8733641d..a8a43da097a 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1,4 +1,4 @@ EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json index e42c7b24c5f..622487a2539 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json index bed35fb5c8c..7e6179fd905 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json index 330ae5d5c17..3f616a3754e 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 4deefe32bbc..bfeb03571f8 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,15 +1,15 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs > out.create.requests.txt --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --nostamp diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index c0257db6330..b3dbd3b782c 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt index c1d4fe55d6e..f8f72b2d600 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.plan.resources.model_serving_endpoints.my_endpoint.action = "create"; json.plan.resources.model_serving_endpoints.my_endpoint.new_state.value.name = "[ENDPOINT_NAME_1]"; diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt index 7b4362bacbe..79919714725 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 1; diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/script b/acceptance/bundle/resources/model_serving_endpoints/basic/script index 972439f6813..48b1f935b0c 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/script +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/script @@ -14,7 +14,7 @@ trap cleanup EXIT # bundle/direct/dresources/permissions.go), so sorting keeps the plan golden # stable across runs. Same approach as secret_scopes/basic and the requests # sorting below. -trace $CLI bundle plan -o json | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Sort access_control_list: the terraform engine stores it as a set whose order # depends on the (unmasked) principal value, so it differs between the local fake @@ -33,7 +33,7 @@ echo "$get_output" | jq '{name, creator}' export ENDPOINT_NAME="test-endpoint-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq print_requests.py //serving-endpoints | gron.py --sort-arrays access_control_list > out.second-requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index 5e30339c21f..7faa6fd31f3 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -7,7 +7,7 @@ title "Configure telemetry out of band" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null title "Remote-only telemetry_config is not drift" -trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' title "Deploy does not remove it" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script index dab44354e0e..03058a0cf1b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite write-only secret and burst_scaling_enabled not r trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The write-only fields are skipped as input_only (confirms the matched rules)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json index 86932f72ce6..2143567be39 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script index 919470c4ca4..a37c37a2df5 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.catalog_name' trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other_catalog" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json index c8e90eb2aad..b48b79aecf3 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json index 3e424ed3fc8..e44a5467be8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script index d48a60c0ace..c27e1968877 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.name' trace update_file.py databricks.yml "name: test-endpoint-$UNIQUE_NAME" "name: test-endpoint-2-$UNIQUE_NAME" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json index 32dd192daab..3e58aa755c8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json index f8dcb1c5510..19ed3fca2b1 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script index 0b68be7c353..db80789d106 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '{route_optimized, id}' > print_requests.py //serving-endpoints > out.first-requests.$DATABRICKS_BUNDLE_ENGINE.json trace update_file.py databricks.yml "route_optimized: false" "route_optimized: true" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json index eeacb3be087..f7b210772cd 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script index 0e31f7a4848..cc76c4428c6 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.schema_name' trace update_file.py databricks.yml "schema_name: default" "schema_name: other_schema" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json index e29d17e4dde..058747f3d60 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script index 7f915920ba1..079bc9a33c9 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.table_name_prefix' trace update_file.py databricks.yml "table_name_prefix: my_table" "table_name_prefix: other_table" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt index 6c3233c9ede..71522c2d87e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.my_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script index 8b420559c3d..8f2e49038ed 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script @@ -8,7 +8,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json index f305ce97719..3f93160a2ab 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script index 6e556d2b1ab..3f942f4d306 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.ai_gateway' trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json index 394377f07f8..32f88590a01 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script index 763379ad9a6..c9b81cd0d6d 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script @@ -17,7 +17,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json index 47e992c1cf9..b4d67db672b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/script b/acceptance/bundle/resources/model_serving_endpoints/update/config/script index 3a28d1dcda2..c4656a110f8 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "name: gpt-4o-mini" "name: gpt-5o-mini" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json index cf8c8b3f0ca..1ff5746e670 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script index 391d369d686..f049becec06 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.email_notifications' # Update email_notifications - change the email address trace update_file.py databricks.yml "user1@example.com" "user2@example.com" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There a bug in TF where it does not actually update the email notifications for a serving endpoint." diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json index 73202db539d..4e1999adb7e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script index 6f69dc2f063..01c0e26eedd 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.tags' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/permissions/_script b/acceptance/bundle/resources/permissions/_script index 8f6a0d9c8cd..df20dfd2fa9 100644 --- a/acceptance/bundle/resources/permissions/_script +++ b/acceptance/bundle/resources/permissions/_script @@ -1,7 +1,7 @@ -trace $CLI bundle validate -o json | nostamp | jq .resources.$RESOURCE.foo.permissions +trace $CLI bundle validate -o json | nostamp.py | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json print_requests() { jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")) and (.path | contains("/api/2.0/bundle") | not))' diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json index 6774f270231..718499a0f66 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.apps.foo": { diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json index 6774f270231..718499a0f66 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.apps.foo": { diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json index bd13e780112..84f6bcd2feb 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.foo": { diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json b/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json index 423fa9d0eb8..5793e74dd90 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.dashboards.foo": { diff --git a/acceptance/bundle/resources/permissions/dashboards/create/script b/acceptance/bundle/resources/permissions/dashboards/create/script index 41f26f63604..2d578386e6a 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/script +++ b/acceptance/bundle/resources/permissions/dashboards/create/script @@ -7,7 +7,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -o json | jq .resources.dashboards.foo.permissions rm out.requests.txt -$CLI bundle plan --output json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json index cc7dfa8e463..35dfda54426 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.database_instances.foo": { diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json index 6ce406b3d28..de8c34ac0ca 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.experiments.foo": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json index 90736028eb4..fd4fc45a826 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.genie_spaces.foo": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index e589a40bd5b..e681e725cde 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json index f048f3e6f48..7185b30f356 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/script b/acceptance/bundle/resources/permissions/jobs/added_remotely/script index e85bcde5379..3b872d6446b 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/script @@ -10,7 +10,7 @@ title "Add permissions out of band" trace $CLI permissions set jobs "$job_id" --json @remote_add.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json index b85fdd51a06..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt index 4b289dbd158..d3c2e3afdca 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 3c8217c195c..529b94814fa 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -1,11 +1,11 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -t green -o json | nostamp | jq .resources +trace $CLI bundle validate -t green -o json | nostamp.py | jq .resources trace errcode $CLI bundle deploy -t green print_requests.py //jobs --nostamp &> out.deploy.requests.json # check plan to ensure there is not drift -trace $CLI bundle plan -o json -t green | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json -t green | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve print_requests.py //jobs --nostamp &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json index 88e1e115a9c..16944e5b4dd 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json index a72e11dc488..74cc6c46453 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/script b/acceptance/bundle/resources/permissions/jobs/delete_one/script index 1f0aa1c80a7..82cd9d0aa94 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/script +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/script @@ -8,7 +8,7 @@ if [ -n "$CLOUD_ENV" ]; then fi rm -f out.requests.txt -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json cleanup() { trace errcode $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json index bd43de04ba7..ceb3d6d81cf 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json index cda86e4ceb4..82626507c1a 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script index cce4d71d0ce..f91513db46e 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script @@ -1,4 +1,4 @@ -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json @@ -11,7 +11,7 @@ title "Delete permissions remotely" trace $CLI permissions set jobs "$job_id" --json @remote_delete.json trace print_requests.py --nostamp //jobs/ -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore.json diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml index 921045dfd09..4ca61be3f36 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml @@ -1,4 +1,3 @@ -# Recording needs a bundle it has seen from the start. This test seeds a state file, -# so recording refuses it. TODO(DMS): drop this once existing state can be -# handed over to the service (see the TODO in dstate.Open). +# Deploy runs as the service principal, later commands as the user - who resolves no deployment of +# its own, so bundle summary reads the job id back null. Recording needs state staged for the reader. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json index 0ec914589b1..d8a4d90a9b0 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json index b85fdd51a06..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json index 2976bb3ebe6..5d47f0cc8ec 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json index 42bd247c77b..993098a0ee6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index e7bfda922b8..f217c1f7185 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json index a2e93e298eb..5c1c6301698 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json index 02ffe2db63a..0402134e14c 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json index 1b25b762637..49058decfa6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index 57ee99c53f4..2c6d989004f 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json index 89b3cf0f302..8cce904f66b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/update/script b/acceptance/bundle/resources/permissions/jobs/update/script index 28f2c121f2b..fb88499c271 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/script +++ b/acceptance/bundle/resources/permissions/jobs/update/script @@ -1,10 +1,10 @@ cp databricks.yml databricks.yml.saved trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json job_id="$(read_id.py job_with_permissions)" @@ -13,21 +13,21 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_one.json trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan @@ -36,7 +36,7 @@ title "Restore original config\n" trace diff.py databricks.yml.saved databricks.yml mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore_original.json trace $CLI bundle plan @@ -44,7 +44,7 @@ trace $CLI bundle plan title "Set permissions: []\n" grep -vE '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml update_file.py databricks.yml '# permissions: [] # EXPLICIT_EMPTY' 'permissions: [] # EXPLICIT_EMPTY' -trace $CLI bundle plan -o json | nostamp > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_set_empty.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json index 151138ddc8b..a783dd0aa59 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json index 268172af1af..ad2ad5b9245 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.models.foo": { diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 0a6935c20b8..8159841e4e3 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json index 41c0fc3ba64..38cad367876 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json index 029250bd21a..56c72cff931 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json index 97493239403..8daa49692ad 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json index 41c0fc3ba64..38cad367876 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json index 240341258e8..e696913fa49 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json index 2ee78e63631..ba600d05faa 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json index db74caabfca..ac3785d797b 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json index 4586f292b32..c3518ec2310 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json index 7e20625e706..6ee152943b7 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json index f4419563280..a43fcd03904 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/script b/acceptance/bundle/resources/permissions/pipelines/update/script index 9ace44addb4..33224b30de3 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/script +++ b/acceptance/bundle/resources/permissions/pipelines/update/script @@ -1,5 +1,5 @@ cp databricks.yml databricks.yml.saved -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_create.json trace $CLI bundle plan @@ -11,14 +11,14 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_delete_one.json trace $CLI bundle plan @@ -26,14 +26,14 @@ trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' --sort > out.requests_delete_all.json trace $CLI bundle plan title "Restore original config\n" mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_restore_original.json trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json index 302eeb3c913..0a0293bf156 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.postgres_projects.foo": { diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json index 6720a13fdaa..f1f44e53bc6 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.sql_warehouses.foo": { diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json b/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json index 71d7ae5cbef..61eb85a3f0b 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 7a443d8cc2d..a2aa063789d 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py --nostamp //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json index eea6e75cb51..d49433933ee 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.vector_search_endpoints.foo": { diff --git a/acceptance/bundle/resources/pipelines/auto-approve/script b/acceptance/bundle/resources/pipelines/auto-approve/script index f0b6c4f9044..c98e2f8e5b3 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/script +++ b/acceptance/bundle/resources/pipelines/auto-approve/script @@ -9,12 +9,12 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.bar.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" title "Assert the job is created" -JOB_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.jobs.foo.id') -$CLI jobs get "${JOB_ID}" | nostamp| jq '{name: .settings.name}' +JOB_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.jobs.foo.id') +$CLI jobs get "${JOB_ID}" | nostamp.py| jq '{name: .settings.name}' title "Remove resources from configuration." trace rm resources.yml diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script index a319f62fb18..79927ff6447 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script @@ -8,5 +8,5 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp.py rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/script b/acceptance/bundle/resources/pipelines/num-workers-zero/script index d23c1e063de..58d7db9fa5b 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/script +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy > out.deploy.txt 2>&1 -trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp.py > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/_script b/acceptance/bundle/resources/pipelines/recreate-keys/_script index 45c53d86743..1121836f497 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/_script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/_script @@ -2,7 +2,7 @@ trace cat databricks.yml touch foo.py touch bar.py trace $CLI bundle plan # should show 'create' -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ppid1=`read_id.py my` @@ -10,7 +10,7 @@ ppid1=`read_id.py my` print_requests() { # Excludes /api/2.0/bundle: a DMS operation path embeds the resource key, so it also # contains "/pipelines". - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp.py rm -f out.requests.txt } @@ -18,14 +18,14 @@ trace print_requests trace update_file.py databricks.yml $CONFIG_UPDATE trace $CLI bundle plan # should show 'recreate' -$CLI bundle plan -o json | nostamp > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests title "Fetch pipeline ID and verify remote state" ppid2=`read_id.py my` -trace $CLI pipelines get $ppid2 | nostamp +trace $CLI pipelines get $ppid2 | nostamp.py title "Verify that original pipeline is gone" trace musterr $CLI pipelines get $ppid1 diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json index 1a34eb33dd4..03f9b803d55 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.my": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index a2bf3b724c7..c15ac5eae0d 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json index d5bd316376b..7c768907755 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.my": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json index 6987c3ad761..da70dac7a17 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/recreate/script b/acceptance/bundle/resources/pipelines/recreate/script index 0a814d08985..247959cb281 100644 --- a/acceptance/bundle/resources/pipelines/recreate/script +++ b/acceptance/bundle/resources/pipelines/recreate/script @@ -8,8 +8,8 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created with catalog" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" # Note: In Terraform provider v1.98.0+, changing catalog no longer triggers recreation. # We switch to using storage location instead, which still triggers recreation. diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json index c07de448831..cb852987772 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 6a6a334022e..267aa0cc3b7 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index d9eb259cbb9..f23ae69f62a 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -21,7 +21,7 @@ rm out.requests.txt title "Fetch pipeline ID and verify remote state" ppid=`read_id.py my` -trace $CLI pipelines get $ppid | nostamp +trace $CLI pipelines get $ppid | nostamp.py rm out.requests.txt title "Destroy the pipeline and verify that it's removed from the state and from remote" diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script index 919fe6e4f4f..f743dc45177 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script @@ -16,7 +16,7 @@ trap cleanup EXIT # resources.json. The field is omitted from JSON when unset (omitempty + # ForceSendFields tracking), so an absent value falls back to "(unset)". get_purge() { - gron.py < .databricks/bundle/default/resources.json | grep 'postgres_branches.branch.state.purge_on_delete' || echo '(unset)' + print_state.py | gron.py | grep 'postgres_branches.branch.state.purge_on_delete' || echo '(unset)' } title "Step 1: deploy with purge_on_delete unset" diff --git a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script index a65cb5ee98b..f86eb93b434 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script +++ b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script @@ -13,7 +13,7 @@ rm -f out.requests.txt title "Change the fork point: source_branch_time is immutable, so this recreates" trace update_file.py databricks.yml "2026-01-01T00:00:00Z" "2026-02-01T00:00:00Z" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy --auto-approve trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_expire_time/script b/acceptance/bundle/resources/postgres_branches/update_expire_time/script index 6c393dc60f0..c35cbbe34f0 100644 --- a/acceptance/bundle/resources/postgres_branches/update_expire_time/script +++ b/acceptance/bundle/resources/postgres_branches/update_expire_time/script @@ -19,7 +19,7 @@ trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z # Only terraform applies it: expire_time is a member of the # expiration oneof, and the direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/script b/acceptance/bundle/resources/postgres_branches/update_protected/script index 1566c885831..7235a56c82c 100755 --- a/acceptance/bundle/resources/postgres_branches/update_protected/script +++ b/acceptance/bundle/resources/postgres_branches/update_protected/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Verify no_change (no changes)" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "is_protected: false" "is_protected: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -51,7 +51,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Restore is_protected to false" trace update_file.py databricks.yml "is_protected: true" "is_protected: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_branches/update_ttl/script b/acceptance/bundle/resources/postgres_branches/update_ttl/script index a450a54d745..ee3e60d1597 100644 --- a/acceptance/bundle/resources/postgres_branches/update_ttl/script +++ b/acceptance/bundle/resources/postgres_branches/update_ttl/script @@ -24,7 +24,7 @@ trace update_file.py databricks.yml "ttl: 604800s" "ttl: 259200s" # creation_time + ttl and never echoes ttl, which the testserver does not model. # The update_mask each engine sends is what this test is about. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/update/script b/acceptance/bundle/resources/postgres_databases/update/script index bdbad2b67c6..22677273763 100644 --- a/acceptance/bundle/resources/postgres_databases/update/script +++ b/acceptance/bundle/resources/postgres_databases/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "postgres_database: initial_db_name" "postgr trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Restore postgres_database to original value" trace update_file.py databricks.yml "postgres_database: renamed_db_name" "postgres_database: initial_db_name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script index 65696066385..43c5a6172d4 100644 --- a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml " suspend_timeout_duration: 300s" "" # Terraform provider masks the whole spec, which the API accepts and then ignores. See # test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -32,4 +32,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//po # fail again on every deploy, terraform applied it and is clean. trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script index f2506c21a01..74bd7ed7077 100755 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "autoscaling_limit_max_cu: 8" "autoscaling_l trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Restore endpoint autoscaling to original value" trace update_file.py databricks.yml "autoscaling_limit_max_cu: 4" "autoscaling_limit_max_cu: 8" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script index ca616cee3f1..e179f96bc5f 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml "statement_timeout: 8000" "statement_timeout # spec.settings.pg_settings are both accepted, while an indexed path is answered with # 400 "Unknown field path in update_mask". trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script index 2f383859007..5b57e845759 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script @@ -26,7 +26,7 @@ title "Plan and deploy" # Both engines apply the change; only the mask they send differs, so out.requests # is the one per-engine file left. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script index 1bdc077e6e1..958d4e2d980 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script @@ -12,7 +12,7 @@ trap cleanup EXIT # resources.json. The field is omitted from JSON when unset (omitempty + # ForceSendFields tracking), so an absent value falls back to "(unset)". get_purge() { - gron.py < .databricks/bundle/default/resources.json | grep purge_on_delete || echo '(unset)' + print_state.py | gron.py | grep purge_on_delete || echo '(unset)' } title "Step 1: deploy with purge_on_delete unset" diff --git a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script index 4d426ae616f..a4f41995e28 100644 --- a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Remove history_retention_duration and re-deploy" trace update_file.py databricks.yml " history_retention_duration: 604800s" "" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -25,4 +25,4 @@ trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace- # record the drift it leaves behind. trace $CLI postgres get-project "${project_name}" | project_fields > out.project.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script index 574a6cf5301..d3aeb95ebfa 100644 --- a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script +++ b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script @@ -25,7 +25,7 @@ trace update_file.py databricks.yml "suspend_timeout_duration: 300s" "suspend_ti # member of the suspension oneof, nested under default_endpoint_settings, and the # direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/script b/acceptance/bundle/resources/postgres_projects/update_display_name/script index 657eca21869..5601e360c32 100755 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/script +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -27,7 +27,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -38,7 +38,7 @@ trace update_file.py databricks.yml "Original Name" "Updated Name" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Restore display_name to original value" trace update_file.py databricks.yml "Updated Name" "Original Name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/update_history_retention/script b/acceptance/bundle/resources/postgres_projects/update_history_retention/script index 7a7ae267a73..d67c658c710 100644 --- a/acceptance/bundle/resources/postgres_projects/update_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/update_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Change history_retention_duration to 259200s and re-deploy" trace update_file.py databricks.yml "history_retention_duration: 604800s" "history_retention_duration: 259200s" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index ce8be6534d3..26f5e3debde 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -2,4 +2,4 @@ Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml index 6de86c5f2b6..01b05580cec 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml @@ -7,3 +7,7 @@ Cloud = false # bind/deploy are engine-agnostic here; run on direct for a single stable output. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# bind and unbind refuse to run when deployment history recording is on, so the +# escape hatch this test covers is not available there yet. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/update/script b/acceptance/bundle/resources/postgres_roles/update/script index 5eea84136a6..d98081efd6f 100644 --- a/acceptance/bundle/resources/postgres_roles/update/script +++ b/acceptance/bundle/resources/postgres_roles/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "createdb: false" "createdb: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Restore attributes.createdb to original value" trace update_file.py databricks.yml "createdb: true" "createdb: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json index e840a5f3161..ce08323b631 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index cef45fe553a..9b8ddfcd2bd 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 15c9c619892..796ce31264a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -17,7 +17,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... Created quality_monitors.monitor1 -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml new file mode 100644 index 00000000000..2acd446f3cd --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml @@ -0,0 +1,3 @@ +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json index cfaff9efccb..ae5c05d00ac 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index a14879d79f1..a51f2233f7d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "output_schema_name: main.qm_test_${UNIQUE_NAME}" "output_schema_name: main.qm_test_${UNIQUE_NAME}_2" trace $CLI bundle plan | contains.py "1 to change" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json index 9b233a1fdbb..eb98b10555c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index ae5c7bd798f..d73c45e3119 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml index 7e03ffedd9a..808576f4791 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml @@ -1,2 +1,6 @@ # this does not work on terraform EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json index b4a49e6fcd7..133e967a99c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.quality_monitors.monitor1": { diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json index bb26c592f1d..2cb6b0913a4 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 4f3596f8375..36e85d2164c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -17,7 +17,7 @@ trap cleanup EXIT envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.txt @@ -26,4 +26,4 @@ trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^ # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/script b/acceptance/bundle/resources/registered_models/aliases_converge/script index ca9dc8f700a..7271aea0090 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/script +++ b/acceptance/bundle/resources/registered_models/aliases_converge/script @@ -15,7 +15,7 @@ title "Plan is a no-op: GET never echoes aliases, so they must not drift" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The config-set aliases are skipped as input_only (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes.aliases' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.aliases' title "Edit the comment: an in-place update, not a recreate" export COMMENT="updated comment" diff --git a/acceptance/bundle/resources/registered_models/basic/script b/acceptance/bundle/resources/registered_models/basic/script index 61dff74182c..c0d1d5e648d 100644 --- a/acceptance/bundle/resources/registered_models/basic/script +++ b/acceptance/bundle/resources/registered_models/basic/script @@ -20,7 +20,7 @@ trap cleanup EXIT deploy_registered_model() { trace $CLI bundle plan trace $CLI bundle deploy - registered_model_id=$($CLI bundle summary --output json | nostamp | jq -r '.resources.registered_models.my_registered_model.id') + registered_model_id=$($CLI bundle summary --output json | nostamp.py | jq -r '.resources.registered_models.my_registered_model.id') trace $CLI registered-models get "${registered_model_id}" | jq '{name, comment, catalog_name, schema_name}' } diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/script b/acceptance/bundle/resources/registered_models/drift/browse_only/script index a3d66f24b0b..84986d5df48 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/script +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite the backend computing browse_only" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only browse_only is skipped as output_only (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes.browse_only' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.browse_only' title "Redeploy is a no-op (no UpdateRegisteredModel call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/auto-approve/script b/acceptance/bundle/resources/schemas/auto-approve/script index f2f2b6cbade..098ef3327b8 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/script +++ b/acceptance/bundle/resources/schemas/auto-approve/script @@ -20,8 +20,8 @@ title "Assert the schema is created" trace $CLI schemas get "${CATALOG_NAME}.${SCHEMA_NAME}" | jq "{full_name, comment}" title "Assert the pipeline is created and uses the schema" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" title "Create a volume in the schema, and add a file to it. This ensures that the schema has some data in it and deletion will fail unless the generated diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/script b/acceptance/bundle/resources/schemas/comment_out_of_band/script index 70871c43a9b..4ac6a877cce 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/script +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/schemas/$SCHEMA" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the schema" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script index fadd886cd48..f4aa3e83214 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/script +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateSchema call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt index 2c6ded2a841..668da79bfb5 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.plan.resources.secret_scopes.my_scope.action = "create"; json.plan.resources.secret_scopes.my_scope.new_state.value.scope = "test-scope-[UNIQUE_NAME]-1"; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt index 44b7e2831ca..f26d6cba67b 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 1; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt index 8745d467735..2782639ba91 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 2; diff --git a/acceptance/bundle/resources/secret_scopes/basic/script b/acceptance/bundle/resources/secret_scopes/basic/script index e8aebc1853f..ebaaf04d898 100755 --- a/acceptance/bundle/resources/secret_scopes/basic/script +++ b/acceptance/bundle/resources/secret_scopes/basic/script @@ -9,7 +9,7 @@ cleanup() { trap cleanup EXIT title "create the secret scope" -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name') @@ -26,7 +26,7 @@ title "update the name of the scope (should recreate)" export SECRET_SCOPE_NAME="test-scope-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Capture API requests for verification. Terraform cleans up ACLs before deleting the scope, but direct does not, hence the difference in requests. @@ -43,4 +43,4 @@ trace $CLI secrets list-acls $scope_name | jq -c '.[]' | sort trace print_requests.py //secrets title "verify there's no persistent drift" -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secrets/remove-expire-time/script b/acceptance/bundle/resources/secrets/remove-expire-time/script index 5f507f1f39a..2f461ba392b 100644 --- a/acceptance/bundle/resources/secrets/remove-expire-time/script +++ b/acceptance/bundle/resources/secrets/remove-expire-time/script @@ -7,13 +7,13 @@ read_state.py secrets secret1 name expire_time title "Remove expire_time and re-deploy" trace update_file.py databricks.yml ' expire_time: "2030-01-01T00:00:00Z"' "" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time title "Plan again to record any drift the removal leaves behind" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/update-expire-time/script b/acceptance/bundle/resources/secrets/update-expire-time/script index 9b915b11936..b9321fd8782 100644 --- a/acceptance/bundle/resources/secrets/update-expire-time/script +++ b/acceptance/bundle/resources/secrets/update-expire-time/script @@ -7,7 +7,7 @@ read_state.py secrets secret1 name expire_time title "Change expire_time and re-deploy" trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 9e94921e2f3..e6cd1b57674 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle plan --var secret_value=initial-secret-value -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.secrets.secret1": { @@ -42,7 +42,7 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Update secret value by passing a different variable value >>> [CLI] bundle plan --var secret_value=updated-secret-value -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -117,20 +117,14 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Verify state does not contain actual secret value >>> print_state.py { - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 2, - "state": { "resources.secrets.secret1": { - "__id__": "main.default.test_secret", - "state": { - "catalog_name": "main", - "comment": "Test secret", - "name": "test_secret", - "schema_name": "default", - "value": "" - } + "__id__": "main.default.test_secret", + "state": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "" + } } - } } diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script index a5c1596e29c..224073934fc 100755 --- a/acceptance/bundle/resources/secrets/update-value/script +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -1,16 +1,16 @@ -trace $CLI bundle plan --var secret_value=initial-secret-value -o json +trace $CLI bundle plan --var secret_value=initial-secret-value -o json | nostamp.py trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value title "Update secret value by passing a different variable value" -trace $CLI bundle plan --var secret_value=updated-secret-value -o json +trace $CLI bundle plan --var secret_value=updated-secret-value -o json | nostamp.py trace $CLI bundle deploy --var secret_value=updated-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value title "Verify state does not contain actual secret value" -{ trace print_state.py | contains.py "!initial-secret-value" "!updated-secret-value"; } || true +{ trace print_state.py | jq .state | contains.py "!initial-secret-value" "!updated-secret-value"; } || true rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script index 58ce3a2f5ad..eaf11e08529 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script @@ -16,7 +16,7 @@ trace $CLI vector-search-endpoints patch-endpoint "${endpoint_name}" --target-qp title "Plan detects drift and proposes update" trace $CLI bundle plan | contains.py "Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged" -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "Deploy restores target_qps to 1" rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index 9f3980b3e80..77b87027314 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -17,7 +17,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle plan --output json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -71,7 +71,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "name": "vector" } ], - "schema_json": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "schema_json": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_uuid": "[UUID]", @@ -87,9 +87,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "direct_access_index_spec.schema_json": { "action": "skip", "reason": "normalized_by_backend", - "old": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "new": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "remote": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "old": "{\"id\":\"integer\",\"vector\":\"array\"}", + "new": "{\"id\":\"integer\",\"vector\":\"array\"}", + "remote": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_uuid": { "action": "skip", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script index b6170f5b613..5761eaae689 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -14,7 +14,7 @@ title "Change endpoint_type (recreates endpoint; index recreate cascade is not y trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED" trace $CLI bundle plan -trace $CLI bundle plan --output json +trace $CLI bundle plan --output json | nostamp.py rm -f out.requests.txt trace $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt index cccd121c387..4b9a961521b 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.catalogs.main_catalog": { diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/script b/acceptance/bundle/resources/volumes/catalog-var-ref/script index 7f1b6190766..5e6053edc06 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/script +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/script @@ -1 +1 @@ -$CLI bundle plan -o json +$CLI bundle plan -o json | nostamp.py diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json index 68e26c30d44..682a73bf1f1 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index 396ffb15155..61d993f6516 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/comment_out_of_band/script b/acceptance/bundle/resources/volumes/comment_out_of_band/script index 3c01424f369..97ce5529786 100644 --- a/acceptance/bundle/resources/volumes/comment_out_of_band/script +++ b/acceptance/bundle/resources/volumes/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/volumes/$VOLUME" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the volume" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan["resources.volumes.volume1"].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json index b5ef8e7d906..cd7a6dfd044 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.volumes.foo": { diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index 536ad6d50b8..3337fd0c6c5 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script index b5403a291f8..d2931527477 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/script +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -21,7 +21,7 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then # seconds after create, so the schema's `properties` change is present # nondeterministically. Prune it with normalize_uc_payload.py before reshaping so # the golden is stable. See #6027 and #6040. - $CLI bundle plan -o json \ + $CLI bundle plan -o json | nostamp.py \ | normalize_uc_payload.py \ | jq '.plan | to_entries | map({resource: .key, changes: (.value.changes // {} | map_values({action, reason}))})' \ >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/run_as/pipelines/_script b/acceptance/bundle/run_as/pipelines/_script index 293600ded7d..d7cf3f76a2a 100644 --- a/acceptance/bundle/run_as/pipelines/_script +++ b/acceptance/bundle/run_as/pipelines/_script @@ -1,5 +1,5 @@ print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp.py rm out.requests.txt } @@ -15,7 +15,7 @@ for target in "${targets[@]}"; do trace $CLI bundle plan -t $target # Debug plan - $CLI bundle plan -o json -t $target | nostamp > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json + $CLI bundle plan -o json -t $target | nostamp.py > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json # Deploy rm -f out.requests.txt diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json index 6bd7cce4936..b29f4242bc9 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json index 98e33a7a82b..fdcbed9bea6 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json index 168d9852fb1..85d6696a669 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json index a489222a9bd..02c785a484d 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json index 5f886cd55ab..f1e14de6938 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json index ef31749eae8..7dab8bf4cbb 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json index 1e780047697..48c2635f87a 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json index 4640930e261..dd0a3e931c2 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index a420f50c421..b5dac5e552c 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index 12db6ca0ba0..e8b7741af95 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,7 +22,7 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json | nostamp > plan.json +$CLI bundle plan --select jobs.foo -o json > plan.json title "bundle deploy --select jobs.foo\n" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) # The deploy reports that --select was used via telemetry. @@ -37,7 +37,7 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > plan-full.json +$CLI bundle plan -o json > plan-full.json title "Full deploy\n" $CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index fe41d5f871a..7166a9e7946 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -8,3 +8,7 @@ EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index dd03baaabb6..bf0f576652e 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index 4b4735b927d..544f8f1e442 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -11,3 +11,7 @@ RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] + +# Dumps plans and replays them via deploy --plan/readplanarg; the replay needs the DMS fields, +# so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/databricks.yml b/acceptance/bundle/state/feature_flags/databricks.yml deleted file mode 100644 index 5134dbcc12c..00000000000 --- a/acceptance/bundle/state/feature_flags/databricks.yml +++ /dev/null @@ -1,7 +0,0 @@ -bundle: - name: test-bundle - -resources: - jobs: - my_job: - name: "my job" diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt deleted file mode 100644 index df55e7c6cee..00000000000 --- a/acceptance/bundle/state/feature_flags/output.txt +++ /dev/null @@ -1,22 +0,0 @@ - -=== a version-3 state recording a feature is rejected (this CLI records no features yet) ->>> errcode [CLI] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information - - -Exit code: 1 - -=== a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2) ->>> [CLI] bundle plan -create jobs.my_job - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.my_job -Files: 6 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> gron.py .databricks/bundle/default/resources.json -json.state_version = 3; diff --git a/acceptance/bundle/state/feature_flags/resources.empty_features.json b/acceptance/bundle/state/feature_flags/resources.empty_features.json deleted file mode 100644 index b20c97aa074..00000000000 --- a/acceptance/bundle/state/feature_flags/resources.empty_features.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "state_version": 3, - "features": {}, - "cli_version": "0.0.0-dev", - "lineage": "test-lineage", - "serial": 1, - "state": {} -} diff --git a/acceptance/bundle/state/feature_flags/resources.with_feature.json b/acceptance/bundle/state/feature_flags/resources.with_feature.json deleted file mode 100644 index b844b098a70..00000000000 --- a/acceptance/bundle/state/feature_flags/resources.with_feature.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "state_version": 3, - "features": { - "future_feature": {} - }, - "cli_version": "0.0.0-dev", - "lineage": "test-lineage", - "serial": 1, - "state": {} -} diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script deleted file mode 100644 index e70b562f2ff..00000000000 --- a/acceptance/bundle/state/feature_flags/script +++ /dev/null @@ -1,11 +0,0 @@ -mkdir -p .databricks/bundle/default - -title "a version-3 state recording a feature is rejected (this CLI records no features yet)" -cp resources.with_feature.json .databricks/bundle/default/resources.json -trace errcode $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" - -title "a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2)" -cp resources.empty_features.json .databricks/bundle/default/resources.json -trace $CLI bundle plan | contains.py "Plan:" -trace $CLI bundle deploy -trace gron.py .databricks/bundle/default/resources.json | grep state_version diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f472..7cf98129ee9 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 3; upgrade the CLI +state version 999 is newer than supported version 2; upgrade the CLI Exit code: 1 diff --git a/acceptance/bundle/state/permission_level_migration/out.plan.direct.json b/acceptance/bundle/state/permission_level_migration/out.plan.direct.json index 33d6194c91e..1492e489619 100644 --- a/acceptance/bundle/state/permission_level_migration/out.plan.direct.json +++ b/acceptance/bundle/state/permission_level_migration/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "test-lineage", "serial": 1, diff --git a/acceptance/bundle/state/permission_level_migration/script b/acceptance/bundle/state/permission_level_migration/script index 32211ebeb49..55fb4aa769f 100644 --- a/acceptance/bundle/state/permission_level_migration/script +++ b/acceptance/bundle/state/permission_level_migration/script @@ -9,10 +9,10 @@ mkdir -p .databricks/bundle/default cp resources.v1.json .databricks/bundle/default/resources.json title "Plan with old v1 state" -trace $CLI bundle plan -o json > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.direct.json title "Deploy (migrates state)" trace $CLI bundle deploy title "Print state after deploy" -trace print_state.py | nostamp --indent 1 +trace print_state.py | nostamp.py --indent 1 diff --git a/acceptance/bundle/summary/modified_status/script b/acceptance/bundle/summary/modified_status/script index 22149b2c76b..fd509c95af0 100644 --- a/acceptance/bundle/summary/modified_status/script +++ b/acceptance/bundle/summary/modified_status/script @@ -1,14 +1,14 @@ title "Initial view of resources without id and modified_status=created" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources trace $CLI bundle deploy title "Post-deployment view of resources with id and without modified_status" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources mv $VARIANT databricks.yml title "Expecting all resources to have modified_status=deleted" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources trace $CLI bundle destroy --auto-approve -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json index 49c706fc239..5607a8fc757 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json index 30918164e2c..4ec8e61237d 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json index b150391beeb..3fb9ec72a4f 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json index a153855fe26..3b20349f64d 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 6b79c6f2251..883a405fa3e 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -7,20 +7,20 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json index 0c6800b20b3..b69ca087696 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json index e637e98dde5..1bb26918106 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/script b/acceptance/bundle/templates/default-python/integration_classic/script index 50b39d6ffce..98c5b97123c 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/script +++ b/acceptance/bundle/templates/default-python/integration_classic/script @@ -17,7 +17,7 @@ trap "trace $CLI bundle destroy -t dev --auto-approve" EXIT trace $CLI bundle validate -t dev trace $CLI bundle validate -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.validate.dev.json -$CLI bundle plan -o json -t dev > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace $CLI bundle summary -t dev trace $CLI bundle summary -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.summary.dev.json @@ -30,7 +30,7 @@ trace $CLI bundle validate -t prod trace $CLI bundle validate -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.validate.prod.json trace diff.py ../out.validate.dev.json ../out.validate.prod.json rm ../out.validate.prod.json -$CLI bundle plan -o json -t prod > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod trace $CLI bundle summary -t prod trace $CLI bundle summary -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.summary.prod.json diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json index 9967e015883..eac2e21abdf 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json index 57a12ae5bb3..c9b06ebd2c5 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json index dd0e402c7df..f68705b05d9 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json index a9db4ff38f4..42a1c6a6351 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index 86272b4a266..c0f2e58f4f4 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -7,19 +7,19 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt trace $CLI bundle deploy -t dev print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/record-deployment-history/out.test.toml b/acceptance/bundle/templates/record-deployment-history/out.test.toml index 27ec2a7fcd6..9921e91a794 100644 --- a/acceptance/bundle/templates/record-deployment-history/out.test.toml +++ b/acceptance/bundle/templates/record-deployment-history/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/templates/record-deployment-history/test.toml b/acceptance/bundle/templates/record-deployment-history/test.toml index a1d3232167a..89b85620c2a 100644 --- a/acceptance/bundle/templates/record-deployment-history/test.toml +++ b/acceptance/bundle/templates/record-deployment-history/test.toml @@ -2,3 +2,7 @@ Env.DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY = "true" # What is under test is what `bundle init` writes, so one engine is enough. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The generated project sets experimental.record_deployment_history; DMS=true permits it so +# the validate step below loads instead of rejecting it. +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 6a259762f65..2db98f4b429 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -10,14 +10,6 @@ EnvMatrix.DMS = ["", "true"] # DMS recording is only supported by the direct engine; it is a no-op on terraform. EnvMatrixExclude.dms_needs_direct = ["DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] -# Saved plans don't carry the deployment stamp. A first plan writes the deployment record, -# so `deploy --plan` creates resources without it and reports drift on the next plan. -EnvMatrixExclude.dms_no_readplan = ["DMS=true", "READPLAN=1"] - -# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory), so -# force it on: the point of this run is to exercise it. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" - # The DMS run asserts the same golden files as the engine runs. EnvRepl.DMS = false diff --git a/bundle/bundle.go b/bundle/bundle.go index 1eb9ce15654..8266db13133 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -317,6 +317,14 @@ func (b *Bundle) WorkspaceClient(ctx context.Context) *databricks.WorkspaceClien return client } +// RecordsDeploymentHistory reports whether this bundle records deployment history with the +// deployment metadata service, from experimental.record_deployment_history or +// DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. +func (b *Bundle) RecordsDeploymentHistory(ctx context.Context) bool { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + return env.RecordsDeploymentHistory(ctx, configured) +} + // SetWorkpaceClient sets the workspace client for this bundle. // This is used to inject a mock client for testing. func (b *Bundle) SetWorkpaceClient(w *databricks.WorkspaceClient) { diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index b7efb4456f9..b59d1b1da02 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -7,4 +7,20 @@ type Deployment struct { // Lock configures locking behavior on deployment. Lock Lock `json:"lock,omitempty"` + + // History reports what the deployment metadata service has recorded for this + // bundle. Output only: it is read from the service for 'bundle summary' and is + // unset when the bundle does not record deployment history. + History *DeploymentHistory `json:"history,omitempty" bundle:"readonly"` +} + +// DeploymentHistory identifies the bundle's deployment in the deployment +// metadata service. +type DeploymentHistory struct { + // DeploymentID is the ID the service assigned to this bundle's deployment. + DeploymentID string `json:"deployment_id,omitempty"` + + // LatestVersionID is the most recent version recorded for the deployment. It is + // unset when the deployment exists but has no version yet. + LatestVersionID string `json:"latest_version_id,omitempty"` } diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 658f1cea819..005c4dfcade 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -53,6 +53,9 @@ type Experimental struct { // RecordDeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. + // + // Only for bundles with no deployed resources yet: DMS becomes the source of + // truth for their state. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..d9bde2b4967 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,41 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. The feature is complete +// but not yet exposed: DMS is dev/staging only, and turning it on makes DMS the +// source of truth for state (irreversible). Setting DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY +// permits it, for CLI tests and DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.RecordDeploymentHistoryEnv(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet; remove this setting from your bundle configuration", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..4f20e61af55 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + recordEnv string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with record env", enabled: true, recordEnv: "true", wantError: false}, + {name: "flag set with empty record env", enabled: true, recordEnv: "", wantError: true}, + {name: "flag unset with record env", enabled: false, recordEnv: "true", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.RecordDeploymentHistoryVariable, tc.recordEnv) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet; remove this setting from your bundle configuration", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ea45903508b..6149bd7b7c9 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, ""); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 65cddeaff19..91cce00f65b 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -156,7 +156,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, ""); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go new file mode 100644 index 00000000000..c08f7b52345 --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -0,0 +1,65 @@ +package metadata + +import ( + "context" + "strconv" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type annotateDeployment struct { + deploymentID string +} + +// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a workspace +// resource points back at the deployment that produced it - how lineage resolves a job to its +// bundle. It runs before the plan, or the stamp would show as drift against local config. +func AnnotateDeployment(deploymentID string) bundle.Mutator { + return &annotateDeployment{deploymentID: deploymentID} +} + +func (m *annotateDeployment) Name() string { + return "metadata.AnnotateDeployment" +} + +func (m *annotateDeployment) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + } + + return nil +} + +type annotateDeploymentVersion struct { + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. +// Separate from AnnotateDeployment because version only exists after CreateVersion runs. +func AnnotateDeploymentVersion(version int64) bundle.Mutator { + return &annotateDeploymentVersion{version: version} +} + +func (m *annotateDeploymentVersion) Name() string { + return "metadata.AnnotateDeploymentVersion" +} + +func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + versionID := strconv.FormatInt(m.version, 10) + + for _, job := range b.Config.Resources.Jobs { + job.Deployment.VersionId = versionID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.VersionId = versionID + } + + return nil +} diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go new file mode 100644 index 00000000000..297ba95f8ee --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -0,0 +1,50 @@ +package metadata + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAnnotateDeploymentVersion(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my-job": { + JobSettings: jobs.JobSettings{ + Deployment: &jobs.JobDeployment{Kind: jobs.JobDeploymentKindBundle}, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Deployment: &pipelines.PipelineDeployment{Kind: pipelines.DeploymentKindBundle}, + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeployment("dep-123"), AnnotateDeploymentVersion(7)) + require.NoError(t, diags.Error()) + + job := b.Config.Resources.Jobs["my-job"].Deployment + assert.Equal(t, "dep-123", job.DeploymentId) + assert.Equal(t, "7", job.VersionId) + // The kind set by AnnotateJobs is preserved. + assert.Equal(t, jobs.JobDeploymentKindBundle, job.Kind) + + pipeline := b.Config.Resources.Pipelines["my-pipeline"].Deployment + assert.Equal(t, "dep-123", pipeline.DeploymentId) + assert.Equal(t, "7", pipeline.VersionId) + assert.Equal(t, pipelines.DeploymentKindBundle, pipeline.Kind) +} diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index c9f5bec3516..3f24c82ee43 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -13,14 +13,23 @@ import ( "github.com/databricks/cli/libs/structs/structvar" ) -const currentPlanVersion = 2 +const currentPlanVersion = 3 type Plan struct { - PlanVersion int `json:"plan_version,omitempty"` - CLIVersion string `json:"cli_version,omitempty"` - Lineage string `json:"lineage,omitempty"` - Serial int `json:"serial,omitempty"` - Plan map[string]*PlanEntry `json:"plan,omitzero"` + PlanVersion int `json:"plan_version,omitempty"` + CLIVersion string `json:"cli_version,omitempty"` + Lineage string `json:"lineage,omitempty"` + Serial int `json:"serial,omitempty"` + + // DMS fields, set only when the bundle records deployment history. The plan targets DeploymentId + // and will create NextVersionId; LastVersionId is the deployment's most recent version at plan + // time. deploy --plan rejects the plan if the deployment moved on from these (see process.go), + // and passes LastVersionId as previous_version_id so the service rejects a stale version too. + DeploymentId string `json:"deployment_id,omitempty"` + NextVersionId string `json:"next_version_id,omitempty"` + LastVersionId string `json:"last_version_id,omitempty"` + + Plan map[string]*PlanEntry `json:"plan,omitzero"` // NotSelected is the number of resources removed by FilterToSelected via the // --select flag. Serialized so the summary survives a deploy from a plan file diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 6b2fffe9a16..af2a7277c77 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -58,6 +58,7 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } } +// Create creates the resource and records its state. func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error { var newID string var remoteState any @@ -83,7 +84,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -126,7 +127,10 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // Drop the state entry so a subsequent failure of Create or WaitAfterDelete // leaves no malformed (empty-ID) entry behind. The next plan will see "no // state" and retry as Create. - err = db.DeleteState(d.ResourceKey) + // + // Recorded as a recreate rather than a delete: if the create below fails, this is the + // operation DMS is left with, and it says the resource is mid-recreate. + err = db.DeleteState(ctx, d.ResourceKey, true) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -168,12 +172,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, false) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -218,7 +222,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -256,11 +260,15 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) } else { - return fmt.Errorf("deleting id=%s: %w", oldID, err) + // The resource is still there, so the history says why rather than leaving its + // operation pending. The state write below is what records a delete that worked. + err = fmt.Errorf("deleting id=%s: %w", oldID, err) + db.RecordFailure(d.ResourceKey, oldID, err) + return err } } - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, false) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -305,7 +313,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -315,11 +323,11 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // saveState saves a state with sensitive fields replaced by a placeholder value so secrets are never written // to disk in plaintext. -func (d *DeploymentUnit) saveState(db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { if err := zeroSensitiveFields(d.Adapter, state); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveState(d.ResourceKey, newID, state, dependsOn) + return db.SaveState(ctx, d.ResourceKey, newID, state, dependsOn) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..798096f6c22 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -61,8 +61,10 @@ type BindResult struct { // Call Finalize to commit the state or Cancel to discard. func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) + // The state is opened without a DMS client, so the writes below record nothing; + // phases.Bind and phases.Unbind refuse to run at all when recording is enabled. var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, ""); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,14 +88,14 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, "") if err != nil { os.Remove(tmpStatePath) return nil, err } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(resourceKey, resourceID, struct{}{}, nil) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +111,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,13 +147,13 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, "") if err != nil { os.Remove(tmpStatePath) return nil, err } - err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +167,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,13 +217,13 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, "") if err != nil { return err } // Delete the main resource - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, false) if err != nil { return err } @@ -235,7 +237,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(key) + err = b.StateDB.DeleteState(ctx, key, false) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 424ae2bdec9..e8910c5eacb 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -43,6 +43,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } + // The state DB records every write with DMS from here on, so the service mirrors the WAL. + // Writes go out on one background goroutine, off the apply path, and are drained below + // once every worker has finished recording. + b.StateDB.RegisterOperationBuffer(b.DmsAsyncOperationClient) + g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { @@ -73,6 +78,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + // Stop resource CRUD once recording state with DMS has failed. + if buf := b.DmsAsyncOperationClient; buf != nil { + if err := buf.Err(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + } + adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) @@ -103,7 +116,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, false) } else { err = d.Destroy(ctx, &b.StateDB) } @@ -134,8 +147,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a recreate reports + // each of its steps. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { + // Empty for a create that never got an ID, and for a recreate whose delete + // step already dropped it. + failedID := b.StateDB.GetResourceID(resourceKey) + b.StateDB.RecordFailure(resourceKey, failedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -162,6 +182,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + // Wait for the uploads and report a failure here, with the deploy's other errors. The phase + // completes the version afterwards and drains again, quietly, so this is not printed twice. + if buf := b.DmsAsyncOperationClient; buf != nil { + if err := buf.Drain(); err != nil { + logdiag.LogError(ctx, err) + } + } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 739ca3bd8e7..f4ec844de4b 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -8,6 +8,7 @@ import ( "maps" "reflect" "slices" + "strconv" "strings" "github.com/databricks/cli/bundle/config" @@ -15,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" @@ -25,6 +27,7 @@ import ( "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/jobs" ) var errDelayed = errors.New("must be resolved after apply") @@ -58,9 +61,11 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. +// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. The plan already +// carries the deployment stamp, except a first deployment's id, which does not exist until deploy: a +// non-empty deploymentID fills it in on any job/pipeline whose loaded entry still lacks one. // StateDB must already be open for write before calling this function. -func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { +func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, deploymentID string) error { b.StateDB.AssertOpenedForWrite() err := b.init(client) @@ -91,6 +96,28 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. if err != nil { return fmt.Errorf("loading plan entry %s: %w", resourceKey, err) } + // Fill in a first deploy's deployment id (see InitForApply doc) on entries that still + // lack one; the version and non-first ids are already in the plan. + if deploymentID != "" { + var stamped bool + switch v := sv.Value.(type) { + case *jobs.JobSettings: + if v.Deployment.DeploymentId == "" { + v.Deployment.DeploymentId = deploymentID + stamped = true + } + case *dresources.PipelineState: + if v.Deployment.DeploymentId == "" { + v.Deployment.DeploymentId = deploymentID + stamped = true + } + } + if stamped { + if err := sv.SyncToJSON(entry.NewState); err != nil { + return fmt.Errorf("%s: stamping deployment into loaded plan: %w", resourceKey, err) + } + } + } b.StateCache.Store(resourceKey, sv) } @@ -128,6 +155,21 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, fmt.Errorf("reading config: %w", err) } + // Record the DMS deployment and version this plan targets. A saved plan carries them so + // deploy --plan can reject a plan the deployment has moved on from, and pass last_version_id + // as previous_version_id. History is set only while recording, so other plans leave these empty. + // configRoot is nil for destroy, which records its version separately. + if configRoot != nil && configRoot.Bundle.Deployment.History != nil { + h := configRoot.Bundle.Deployment.History + next, err := dms.NextVersion(h.LatestVersionID) + if err != nil { + return nil, fmt.Errorf("computing next deployment version: %w", err) + } + plan.DeploymentId = h.DeploymentID + plan.LastVersionId = h.LatestVersionID + plan.NextVersionId = strconv.FormatInt(next, 10) + } + b.Plan = plan g, err := makeGraph(plan) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go new file mode 100644 index 00000000000..c0d4ffb2a7f --- /dev/null +++ b/bundle/direct/dstate/dms.go @@ -0,0 +1,58 @@ +package dstate + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" +) + +// RecordedState is what the CLI serializes into the DMS operation's state field. It wraps the +// config so depends_on survives the round trip: DMS has no field for dependency edges, and +// nesting them in the config would collide with resource fields of the same name. +type RecordedState struct { + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` +} + +// applyDMSState replaces the file-derived resource state with what DMS recorded. DMS owns the +// resource set outright, so this runs on every recorded open and the file's copy is never read +// back as the truth: an empty set means the service tracks nothing, whether because the deploy +// created nothing or because the deployment is gone. The caller holds db.mu. +func (db *DeploymentState) applyDMSState(recorded []dms.Resource) error { + // Built first and assigned together, so a malformed envelope leaves the state as it was + // rather than half replaced. + resources := make(map[string]ResourceEntry, len(recorded)) + stateIDs := make(map[string]string, len(recorded)) + for _, res := range recorded { + entry, err := stateEntry(res) + if err != nil { + return err + } + resources[res.Key] = entry + stateIDs[res.Key] = entry.ID + } + + db.Data.State = resources + db.stateIDs = stateIDs + return nil +} + +// stateEntry unwraps the envelope the write path recorded for a resource. +func stateEntry(res dms.Resource) (ResourceEntry, error) { + var recorded RecordedState + if res.State != "" { + // The service stores state as an opaque string, so it arrives as the serialized + // envelope the write side sent. + if err := json.Unmarshal([]byte(res.State), &recorded); err != nil { + return ResourceEntry{}, fmt.Errorf("interpreting state recorded for %s: %w", res.Key, err) + } + } + + return ResourceEntry{ + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + }, nil +} diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go new file mode 100644 index 00000000000..8edcf563ee8 --- /dev/null +++ b/bundle/direct/dstate/dms_test.go @@ -0,0 +1,100 @@ +package dstate + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyDMSState(t *testing.T) { + // The service stores state as an opaque string, so the envelope the write path sent + // arrives verbatim. + envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` + + // What the file loaded, for the cases that check it is replaced or survives. + fileState := map[string]ResourceEntry{ + "resources.jobs.bar": {ID: "file-id", State: json.RawMessage(`{"name":"from-file"}`)}, + } + + tests := []struct { + name string + existing map[string]ResourceEntry + recorded []dms.Resource + want map[string]ResourceEntry + wantErr string + }{ + { + name: "the envelope is unwrapped, depends_on and all", + recorded: []dms.Resource{ + {Key: "resources.jobs.foo", ID: "123", State: envelope}, + {Key: "resources.pipelines.bar", ID: "456"}, + }, + // depends_on comes back from the envelope, so a bundle whose local state was + // wiped still has the edges needed for delete ordering. + want: map[string]ResourceEntry{ + "resources.jobs.foo": { + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + }, + "resources.pipelines.bar": {ID: "456"}, + }, + }, + { + name: "what DMS holds replaces what the file loaded", + existing: fileState, + recorded: []dms.Resource{{Key: "resources.jobs.foo", ID: "dms-id", State: `{"state":{"name":"from-dms"}}`}}, + want: map[string]ResourceEntry{"resources.jobs.foo": {ID: "dms-id", State: json.RawMessage(`{"name":"from-dms"}`)}}, + }, + { + // Nil is what a list holding nothing returns, and it means a successful deploy of + // nothing rather than missing data. + name: "nothing recorded empties the state", + existing: fileState, + want: map[string]ResourceEntry{}, + }, + { + // The good resource comes first, so the error lands mid-way: what the file loaded + // has to survive whole rather than end up half replaced. + name: "a malformed envelope leaves the state as it was", + existing: fileState, + recorded: []dms.Resource{ + {Key: "resources.jobs.ok", ID: "999", State: `{"state":{"name":"ok"}}`}, + {Key: "resources.jobs.foo", ID: "123", State: "not json"}, + }, + wantErr: "interpreting state recorded for resources.jobs.foo", + want: fileState, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var db DeploymentState + db.Data.State = tt.existing + db.stateIDs = stateIDsOf(tt.existing) + + err := db.applyDMSState(tt.recorded) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.want, db.Data.State) + assert.Equal(t, stateIDsOf(tt.want), db.stateIDs) + }) + } +} + +// stateIDsOf is the id index the state keeps alongside its entries. +func stateIDsOf(entries map[string]ResourceEntry) map[string]string { + ids := make(map[string]string, len(entries)) + for key, entry := range entries { + ids[key] = entry.ID + } + return ids +} diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..dccb06686a5 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,8 +3,6 @@ package dstate import ( "encoding/json" "fmt" - "slices" - "strings" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structpath" @@ -15,24 +13,6 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. - if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) - for name := range db.Features { - features = append(features, name) - } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) - } - if db.StateVersion == currentStateVersion { return nil } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index af2e2638eb3..4dcf53a765a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/google/uuid" ) @@ -28,35 +29,20 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade - // - // A featureStateVersion state with no features is equivalent to - // currentStateVersion, but we deliberately do not flip the on-disk version down - // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. - featureStateVersion = 3 - - // supportedStateVersion is the highest schema version this CLI can read. It is - // normally equal to currentStateVersion — the version this CLI reads is the - // version it writes — and exceeds it only during a two-phase version bump like - // the current feature-flag scaffolding, where this CLI reads (but does not - // write) featureStateVersion. A state newer than this is rejected as too new. - supportedStateVersion = featureStateVersion + // supportedStateVersion is the highest schema version this CLI can read: the + // version it writes. A state newer than this is rejected as too new. + supportedStateVersion = currentStateVersion ) -// featuresDocURL is the single documentation page describing deployment state -// feature flags. It is shown when a state records a feature this CLI does not -// support; it is a fixed link for all features. The #state-features anchor points -// at the feature table; if it ever breaks, the user still lands on the page. -const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" +// featureRecordDeploymentHistory marks a state whose resources are also recorded with the +// deployment metadata service. Both stores are kept in step, so the marker is what tells a +// reader the two already agree. A CLI that does not know the name refuses the state rather +// than deploying over a deployment it would leave the service out of step with. +// +// The marker is sticky: once a deployment is recorded, the service holds resources that a +// CLI which is not recording must not touch. So turning recording off does not clear it, and +// deploying such a state without recording is refused (see RequiresDeploymentHistory). +const featureRecordDeploymentHistory = "record_deployment_history" // errStaleWAL is returned when the WAL serial is behind the expected serial. // The caller should delete the stale WAL and proceed normally. @@ -70,6 +56,10 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // operationBuffer records each state write with DMS. Nil unless the bundle records deployment + // history, in which case RegisterOperationBuffer installs it once the version exists. + operationBuffer *dms.OperationBuffer } type Header struct { @@ -84,10 +74,9 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. This CLI writes no features; it only reads the field to detect a state - // that depends on features it lacks and refuse it (see migrateState). It is a - // map so a future CLI can attach per-feature data without reshaping the state. - // Empty/omitted for states that use no features. + // value. It is read to detect a state that depends on features this CLI lacks and + // refuse it (see migrateState). It is a map so a future CLI can attach per-feature + // data without reshaping the state. Empty/omitted for states that use no features. Features map[string]struct{} `json:"features,omitempty"` } @@ -111,6 +100,51 @@ type WALEntry struct { Value *ResourceEntry `json:"v,omitempty"` // nil means delete } +// RegisterOperationBuffer has every subsequent state write recorded through buf, so what the +// service holds mirrors the WAL. A nil buffer records nothing. Called once the version exists, +// which is why it is not an Open option. +func (db *DeploymentState) RegisterOperationBuffer(buf *dms.OperationBuffer) { + if buf == nil { + return + } + + db.mu.Lock() + defer db.mu.Unlock() + db.operationBuffer = buf +} + +// RecordFailure records that a resource did not apply, so the history says why rather than +// leaving the resource out. resourceID is the id it had before the failure. +func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause error) { + r := db.recorder() + if r == nil { + return + } + + // The service refuses a failure that leaves a live resource described by nothing, so re-state + // what it still has. An empty resourceID means there is nothing left: a create that never + // landed, or a recreate whose delete already went through. + var recorded json.RawMessage + if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { + var err error + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + // Nothing the caller can act on, so record the failure without the state. + recorded = nil + } + } + + r.RecordFailure(resourceKey, resourceID, recorded, cause) +} + +// recorder reads the buffer under db.mu, which guards it, and returns nil when the bundle does +// not record deployment history. +func (db *DeploymentState) recorder() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + return db.operationBuffer +} + func NewDatabase(lineage string, serial int) Database { return Database{ Header: Header{ @@ -123,47 +157,81 @@ func NewDatabase(lineage string, serial int) Database { } } -func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// SaveState records the resource's state after an operation was applied to it. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpenedForWrite() - db.mu.Lock() - defer db.mu.Unlock() - - if db.Data.State == nil { - db.Data.State = make(map[string]ResourceEntry) - } jsonMessage, err := json.Marshal(state) if err != nil { return err } - entry := ResourceEntry{ ID: newID, State: json.RawMessage(jsonMessage), DependsOn: dependsOn, } + db.mu.Lock() + if db.Data.State == nil { + db.Data.State = make(map[string]ResourceEntry) + } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err == nil { db.stateIDs[key] = newID } - return err + buf := db.operationBuffer + db.mu.Unlock() + + if err != nil { + return err + } + if buf == nil { + return nil + } + + // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, + // and outside db.mu because recording waits when the service is behind - waiting under the + // lock would hold up every other resource's write. + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + buf.RecordOperation(ctx, key, false, newID, recorded) + + return nil } -func (db *DeploymentState) DeleteState(key string) error { +// DeleteState drops the resource's state entry: the resource is gone. inProgress records the +// operation as unfinished, which is what the first half of a recreate wants - an interrupted +// deploy must not leave the resource described as finished. +func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgress bool) error { db.AssertOpenedForWrite() - db.mu.Lock() - defer db.mu.Unlock() + db.mu.Lock() if db.Data.State == nil { + db.mu.Unlock() return nil } - + // Read before the delete: DMS needs the id to say which resource went away. + deletedID := db.stateIDs[key] err := appendJSONLine(db.walFile, WALEntry{Key: key}) if err == nil { delete(db.stateIDs, key) } - return err + buf := db.operationBuffer + db.mu.Unlock() + + if err != nil { + return err + } + + // State is nil: the resource no longer exists. Recorded outside the lock for the + // same reason as SaveState. + if buf != nil { + buf.RecordOperation(ctx, key, inProgress, deletedID, nil) + } + + return nil } func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { @@ -201,6 +269,18 @@ func (db *DeploymentState) StateCLIVersion() string { return db.Data.CLIVersion } +// RequiresDeploymentHistory reports whether the state depends on the deployment metadata +// service recording it. Deploying such a state without recording would leave the service +// holding a deployment that no longer matches, so the caller refuses instead. +func (db *DeploymentState) RequiresDeploymentHistory() bool { + db.AssertOpenedForReadOrWrite() + db.mu.Lock() + defer db.mu.Unlock() + + _, ok := db.Data.Features[featureRecordDeploymentHistory] + return ok +} + // GetOrInitLineage returns the deployment lineage, generating and storing a new // one if the state does not have one yet. It is the single place the lineage is // initialized, shared so the direct deployment engine (when it writes state, via @@ -230,7 +310,12 @@ type ( WithWrite bool ) -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +// Open reads the deployment state from disk, recovering the WAL when withRecovery is set. +// With a non-nil dmsClient the resources come from DMS instead, and dmsDeploymentID is the id +// the service holds for the deployment, empty before its first recorded deploy; lineage and +// serial still come from the file, since that is what the write path increments. Open only reads +// through the client - RegisterOperationBuffer installs the write path, once a version exists. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient *dms.Client, dmsDeploymentID string) error { db.mu.Lock() defer db.mu.Unlock() @@ -238,7 +323,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W panic(fmt.Sprintf("state already opened: %v, cannot open %v", db.Path, path)) } - err := db.unlockedOpen(ctx, path, withRecovery, withWrite) + err := db.unlockedOpen(ctx, path, withRecovery, withWrite, dmsClient, dmsDeploymentID) if err != nil { // A failed open must leave the receiver closed. unlockedOpen assigns // db.Path before every fallible step, so without this the receiver stays @@ -260,7 +345,7 @@ func (db *DeploymentState) reset() { db.stateIDs = nil } -func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient *dms.Client, dmsDeploymentID string) error { db.Path = path data, err := os.ReadFile(db.Path) if err != nil { @@ -281,14 +366,22 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe } walPath := db.Path + walSuffix - _, err = os.Stat(walPath) - switch { - case errors.Is(err, fs.ErrNotExist): - // no WAL, nothing to do - case err != nil: - return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) - default: // WAL exists - if withRecovery { + _, statErr := os.Stat(walPath) + walExists := statErr == nil + if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) { + return fmt.Errorf("failed to stat WAL file %s: %w", walPath, statErr) + } + + // A recorded deployment gets its resources from the service (applyDMSState below), so the WAL - + // a local log of an uncommitted deploy - is not merged. A leftover from a crash or a declined + // deploy is removed so the next write's exclusive create of the WAL does not fail. Without + // recording, replay the WAL as usual. + if walExists { + if dmsClient != nil { + if err := os.Remove(walPath); err != nil { + return fmt.Errorf("removing WAL file %s: %w", walPath, err) + } + } else if withRecovery { if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } @@ -301,6 +394,42 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("migrating state %s: %w", path, err) } + if dmsClient != nil { + // The service is the source of truth for a recorded deployment; local state is irrelevant. + // This state was pulled from the workspace, so it reflects the remote and not just a local + // cache, and a leftover WAL was discarded above rather than replayed. It records the feature + // once recording owns it, so a recorded deployment carries no resources here (a tombstone) + // and applyDMSState below loads the service's. A state that still tracks resources without + // the feature is a deployment made before recording: treating the service as authoritative + // would create those resources a second time, so refuse it (adopting one is a followup). + if _, recorded := db.Data.Features[featureRecordDeploymentHistory]; !recorded && len(db.Data.State) > 0 { + return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again`) + } + + // Mark the state as depending on the service so this CLI keeps recording it (see + // RequiresDeploymentHistory). unlockedSave then writes the header alone. Bumping the state + // version so older CLIs refuse a recorded state is a followup. + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}, 1) + } + db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + + if dmsDeploymentID != "" { + resources, err := dmsClient.ListResources(ctx, dmsDeploymentID) + if err != nil { + return err + } + if err := db.applyDMSState(resources); err != nil { + return err + } + } + } + if withWrite { if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) @@ -405,6 +534,7 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) if header.Serial > expectedSerial { return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } + newSerial = header.Serial newCLIVersion = header.CLIVersion } else { @@ -604,7 +734,7 @@ func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.Export // the WAL. A torn write would therefore leave a state file that Open rejects // next to an intact WAL it never reads. func (db *DeploymentState) unlockedSave() error { - data, err := json.MarshalIndent(db.Data, "", " ") + data, err := json.MarshalIndent(db.dataToPersist(), "", " ") if err != nil { return err } @@ -640,6 +770,20 @@ func (db *DeploymentState) unlockedSave() error { return nil } +// dataToPersist returns what the state file should hold: db.Data itself, unless the deployment +// records its history, in which case it is the header alone. The service holds those resources and +// is where they are read back from, so writing them here too would be a second, never-read copy. +// Returns a copy, leaving the in-memory state - what the rest of the deploy works from - untouched. +func (db *DeploymentState) dataToPersist() Database { + data := db.Data + if _, recorded := data.Features[featureRecordDeploymentHistory]; !recorded { + return data + } + + data.State = map[string]ResourceEntry{} + return data +} + func appendJSONLine(file *os.File, obj any) error { data, err := json.Marshal(obj) if err != nil { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 34ed9ff1c5a..8be586583d7 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -21,14 +21,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, "")) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -38,7 +38,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) mustFinalize(t, &db) _, err := os.Stat(path) @@ -94,10 +94,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true)) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "") }) mustFinalize(t, &db) } @@ -115,12 +115,12 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) + require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, "")) assert.Equal(t, build.GetInfo().Version, reopened.Data.CLIVersion) assert.Equal(t, 2, reopened.Data.Serial) mustFinalize(t, &reopened) @@ -142,7 +142,7 @@ func TestHeaderOnlyWALDoesNotUpdateCLIVersion(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, "")) assert.Equal(t, "0.1.2", recovered.Data.CLIVersion, "a header-only WAL wrote no state, so the version must not move") assert.Equal(t, 1, recovered.Data.Serial) mustFinalize(t, &recovered) @@ -154,12 +154,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, "")) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -175,60 +175,28 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, "")) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) mustFinalize(t, &recovered) } -// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a -// featureStateVersion state with no features is accepted as-is — the on-disk version -// is left at featureStateVersion, not flipped down to currentStateVersion — and that -// a featureStateVersion state recording any feature is refused. This is scaffolding -// for the deferred version bump, special-cased to featureStateVersion only (see the -// featureStateVersion doc comment). -// -// When the baseline is actually bumped to featureStateVersion, this special case must -// go away. This test is the forcing function: it fails once featureStateVersion is -// removed, making the author decide what the post-bump behavior should be. -func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { - // The special case applies to featureStateVersion (3) only. - require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case") - require.Equal(t, 3, featureStateVersion) - - empty := &Database{Header: Header{StateVersion: featureStateVersion}} - require.NoError(t, migrateState(empty)) - assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - - // v3 that records a feature is refused: this CLI does not understand features. - withFeature := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{"future_feature": {}}, - }} - err := migrateState(withFeature) - require.Error(t, err) - assert.Contains(t, err.Error(), "requires features this CLI does not support") - assert.Contains(t, err.Error(), "future_feature") - assert.Contains(t, err.Error(), "upgrade to the latest CLI version") - assert.Contains(t, err.Error(), featuresDocURL) -} - func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db2.DeleteState("jobs.my_job")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", false)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, "")) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -240,7 +208,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, "")) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -252,12 +220,12 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, "")) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } @@ -271,13 +239,13 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600)) var db DeploymentState - require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) assert.Empty(t, db.Path) // Once the state file is readable, the same receiver opens without panicking. seed := `{"state_version":2,"cli_version":"0.1.2","lineage":"test-lineage","serial":1,"state":{}}` require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, "")) assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index ea7ae1e51b6..764397d5fc3 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt/resourcestate" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/structs/structvar" ) @@ -50,6 +51,13 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // DMS carries deployment-history recording across the phases; nil unless the bundle records + // history. DmsApiClient is set where the state is opened. The deployment id and version live + // in bundle.deployment.history, not here. DmsAsyncOperationClient is opened once the deploy + // phase creates the version, and Apply drains it before returning. + DmsApiClient *dms.Client + DmsAsyncOperationClient *dms.OperationBuffer } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/env/dms.go b/bundle/env/dms.go new file mode 100644 index 00000000000..1e5002e0960 --- /dev/null +++ b/bundle/env/dms.go @@ -0,0 +1,23 @@ +package env + +import "context" + +// RecordDeploymentHistoryVariable enables recording without setting the config field. +// Exists for CLI acceptance tests so the whole bundle suite runs with recording on. +// Deliberately undocumented; see validate.ValidateRecordDeploymentHistory. +const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" + +// RecordDeploymentHistoryEnv reports whether the environment turns on deployment history +// recording. Only "true" turns it on: anything else - including a typo like "TRUE" or "yes" - +// leaves a gated feature off rather than silently enabling it. Setting it is also what permits +// the otherwise-gated experimental.record_deployment_history (see validate.ValidateRecordDeploymentHistory). +func RecordDeploymentHistoryEnv(ctx context.Context) bool { + value, _ := get(ctx, []string{RecordDeploymentHistoryVariable}) + return value == "true" +} + +// RecordsDeploymentHistory reports whether recording is on, from config or env var. +// Single predicate for all recording code paths; keeps them in sync. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + return configured || RecordDeploymentHistoryEnv(ctx) +} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go new file mode 100644 index 00000000000..15bfe0e63f0 --- /dev/null +++ b/bundle/env/dms_test.go @@ -0,0 +1,41 @@ +package env + +import ( + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" +) + +func TestRecordDeploymentHistoryEnv(t *testing.T) { + for _, tc := range []struct { + value string + want bool + }{ + {"true", true}, + {"", false}, + {"0", false}, + {"false", false}, + // Only "true" counts, so a near miss leaves recording off. + {"1", false}, + {"TRUE", false}, + {"yes", false}, + } { + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, tc.value) + assert.Equal(t, tc.want, RecordDeploymentHistoryEnv(ctx), "value %q", tc.value) + } +} + +func TestRecordDeploymentHistoryEnvUnset(t *testing.T) { + assert.False(t, RecordDeploymentHistoryEnv(t.Context())) +} + +func TestRecordsDeploymentHistory(t *testing.T) { + // The bundle setting alone is enough, and so is the environment; the env var + // exists so the acceptance suite can record without touching every databricks.yml. + assert.True(t, RecordsDeploymentHistory(t.Context(), true)) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) + + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, "true") + assert.True(t, RecordsDeploymentHistory(ctx, false)) +} diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index e8b382b370d..7afa577ba8c 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -205,7 +205,7 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 81f110f5095..b3023e83991 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -33,6 +33,11 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en }() if engine.IsDirect() { + if b.RecordsDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("bind is not supported for a bundle that records deployment history")) + return + } + // Direct engine: import into temp state, run plan, check for changes // This follows the same pattern as terraform import groupName, ok := terraform.TerraformToGroupName[opts.ResourceType] @@ -94,7 +99,8 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en } } else { // Terraform engine: use terraform import - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Import(opts), @@ -129,6 +135,11 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { + if b.RecordsDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("unbind is not supported for a bundle that records deployment history")) + return + } + groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType @@ -141,7 +152,8 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r return } } else { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Unbind(bundleType, tfResourceType, resourceKey), diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f16088e4518..c22a6c1a4d7 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -26,9 +26,11 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) var deployApprovalGroups = []approvalGroup{ @@ -110,7 +112,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st return } - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, statemgmt.Load(state), metadata.Compute(), metadata.Upload(), @@ -168,7 +171,8 @@ func logDeploySummary(ctx context.Context, b *bundle.Bundle, plan *deployplan.Pl // It also cleans up the artifacts directory and transforms wheel tasks. // It is called by only "bundle deploy". func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]libraries.LocationToUpdate) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, artifacts.CleanUp(), libraries.Upload(libs), ) @@ -179,12 +183,13 @@ func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]li // stateEngine is the engine the resolved state file uses; requestedEngine is // what bundle.engine / DATABRICKS_BUNDLE_ENGINE asked for and may differ (used // only by the post-deploy migration check). -func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan) { +func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan, dmsDeployment *bundledeployments.Deployment) { log.Info(ctx, "Phase: deploy") // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, scripts.Execute(config.ScriptPreDeploy), lock.Acquire(lock.GoalDeploy), ) @@ -195,7 +200,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } // lock is acquired here + // + // The version is created only after approval; CompleteVersion is deferred before + // lock.Release and no-ops until then. defer func() { + if _, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -209,7 +220,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Upload all source files and built artifacts as a single immutable snapshot. // snapshot.Upload() sets workspace.snapshot_path; the variable-resolution // pass expands ${workspace.snapshot_path} placeholders written by translate_paths. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, snapshot.Upload(), mutator.ResolveVariableReferencesOnlyResources("workspace"), ) @@ -246,7 +258,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } }() - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), @@ -259,6 +272,24 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } planFromFile := plan != nil + if b.DeploymentBundle.DmsApiClient != nil { + // Create the deployment before planning so it exists to stamp. + createOrUpdateDeployment(ctx, b, dmsDeployment) + if logdiag.HasError(ctx) { + return + } + // version_id is annotated at plan time. Once the deployment exists, stamp its id onto the + // config so RunPlan carries it into the plan; the id is unknown at plan time for a first + // deploy, so this is where a first deploy picks it up. + if !planFromFile { + deploymentID, _ := deploymentAndNextVersion(b) + bundle.ApplySeqContext(ctx, b, metadata.AnnotateDeployment(deploymentID)) + if logdiag.HasError(ctx) { + return + } + } + } + if plan == nil { // State is already open for read by process.go (for direct engine) plan = RunPlan(ctx, b, stateEngine) @@ -280,8 +311,10 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } if planFromFile { - // Initialize DeploymentBundle for applying the loaded plan - err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) + // The loaded plan already carries the stamps, except a first deploy's deployment id, which + // does not exist until now; InitForApply fills that in. Non-recording deploys pass "". + deploymentID, _ := deploymentAndNextVersion(b) + err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan, deploymentID) if err != nil { logdiag.LogError(ctx, err) return @@ -295,17 +328,35 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - haveApproval, err := approvalForDeploy(ctx, b, plan) + haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) + if !haveApproval { + // No version was created, so the deferred CompleteVersion is a no-op and the version + // number is left for the next deploy. Both the user declining and a console that + // cannot prompt land here. + if approvalErr != nil { + logdiag.LogError(ctx, approvalErr) + return + } + cmdio.LogString(ctx, "Deployment cancelled!") + return + } + + // Create the version the plan was stamped with, staging an operation for every resource + // it touches. Doing it here rather than before the prompt means a declined deploy never + // claims a version number. + staged, err := stagedOperations(plan) if err != nil { logdiag.LogError(ctx, err) return } - if haveApproval { - deployCore(ctx, b, plan, stateEngine) - } else { - cmdio.LogString(ctx, "Deployment cancelled!") + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + logdiag.LogError(ctx, err) return } + deploymentID, versionID := deploymentAndNextVersion(b) + logDeploymentVersion(ctx, b, deploymentID, versionID) + + deployCore(ctx, b, plan, stateEngine) if logdiag.HasError(ctx) { return @@ -358,7 +409,8 @@ func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *d // b.Select is rejected for the terraform engine in ProcessBundleRet, so it is // never set here. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("deploy")), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 201c3928e28..37fa4645a13 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -158,6 +159,21 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } + // Complete version before deleting remote files; the deployment node is under statePath. + completed, err := drainOperationsAndCompleteVersion(ctx, b, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + // A completed destroy's resources are gone, so its deployment record is deleted too. + if completed { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + return + } + } + bundle.ApplyContext(ctx, b, files.Delete()) if !logdiag.HasError(ctx) && b.Quiet < bundle.QuietAll { @@ -196,12 +212,26 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } + // DMS recording of this destroy: the version is created after approval, so a cancelled + // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once + // destroyCore has completed the version. defer func() { + completed, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)) + if err != nil { + logdiag.LogError(ctx, err) + } else if completed { + // A completed destroy's resources are gone, so its deployment record is deleted too. + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + } + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() if !engine.IsDirect() { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // We need to resolve artifact variable (how we do it in build phase) // because some of the to-be-destroyed resource might use this variable. // Not resolving might lead to terraform "Reference to undeclared resource" error @@ -253,6 +283,22 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } + // Record a destroy version now that it is approved and the state WAL is open, but only + // under a deployment that already exists: a missing one was already cleaned up (or the + // bundle does not record history). Destroy never creates or updates the deployment - it + // is about to be deleted. + deploymentID, _ := deploymentAndNextVersion(b) + if b.DeploymentBundle.DmsApiClient != nil && deploymentID != "" { + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := startVersion(ctx, b, dms.VersionTypeDestroy, staged); err != nil { + logdiag.LogError(ctx, err) + return + } + } destroyCore(ctx, b, plan, engine) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go new file mode 100644 index 00000000000..1ccf0439b63 --- /dev/null +++ b/bundle/phases/dms.go @@ -0,0 +1,249 @@ +package phases + +import ( + "context" + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// stagedOperations lists the resources the plan will touch, for CreateVersion to stage an +// operation each. Skipped and undefined actions are left out: nothing is applied for them, so +// their operations would stay pending and the service would hold no state for them. +func stagedOperations(plan *deployplan.Plan) ([]dms.StagedOperation, error) { + actions := plan.GetActions() + staged := make([]dms.StagedOperation, 0, len(actions)) + for _, action := range actions { + if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Undefined { + continue + } + actionType, err := actionToSDK(action.ActionType) + if err != nil { + return nil, fmt.Errorf("%s: %w", action.ResourceKey, err) + } + staged = append(staged, dms.StagedOperation{ + ResourceKey: action.ResourceKey, + ActionType: actionType, + }) + } + return staged, nil +} + +// actionToSDK maps a deployplan action to the DMS action type a staged operation records. +// Only actions that mutate a resource are recordable; Skip and Undefined are rejected +// rather than silently coerced. +func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) + } +} + +// deploymentAndNextVersion reads, from the history the recording set, the deployment id and the version +// this run will create (one past the deployment's most recent). Both zero when nothing is recorded +// - the bundle records no history, so version 0 marks "no version" for logDeploymentVersion. +func deploymentAndNextVersion(b *bundle.Bundle) (string, int64) { + h := b.Config.Bundle.Deployment.History + if h == nil { + return "", 0 + } + // LatestVersionID comes from the service as an integer, so this does not fail in practice. + version, err := dms.NextVersion(h.LatestVersionID) + if err != nil { + version = 1 + } + return h.DeploymentID, version +} + +// createOrUpdateDeployment creates the deployment on a first deploy, or updates the metadata this +// run changed. current is the record the service holds (nil before the first recorded deploy), +// diffed to mask the update down to what changed. Runs before the plan, so the deployment exists +// to be stamped onto the resources; the version it will create is settled in process.go. +func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) { + db := &b.DeploymentBundle + metadata := deploymentMetadata(b) + deploymentID, _ := deploymentAndNextVersion(b) + switch mask := metadata.StaleFields(current); { + case deploymentID == "": + id, err := db.DmsApiClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) + return + } + deploymentID = id + case mask != "": + if err := db.DmsApiClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) + return + } + } + + // A first deploy had no deployment to read at startup, so its id enters the history here. + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + if b.Config.Bundle.Deployment.History == nil { + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{} + } + b.Config.Bundle.Deployment.History.DeploymentID = deploymentID + }) +} + +// startVersion claims the version the run settled on and opens the buffer that records +// each state write under it. Called after approval, so a declined deploy never claims a number. +// A no-op when the bundle does not record deployment history. +func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.VersionType, staged []dms.StagedOperation) error { + db := &b.DeploymentBundle + if db.DmsApiClient == nil { + return nil + } + deploymentID, versionID := deploymentAndNextVersion(b) + + // The version this run creates is one past the deployment's most recent, so the previous is + // one below it - empty for the first version. + previousVersionID := "" + if versionID > 1 { + previousVersionID = strconv.FormatInt(versionID-1, 10) + } + + // The server rejects this unless the version number exceeds last_version_id and + // previous_version_id matches it, which is what makes claiming the number up front + // safe: a deploy that took it in the meantime is reported, not overwritten. + var gitInfo *bundledeployments.GitInfo + if git := b.Config.Bundle.Git; git.Branch != "" || git.Commit != "" || git.OriginURL != "" { + gitInfo = &bundledeployments.GitInfo{ + Branch: git.Branch, + Commit: git.Commit, + OriginUrl: git.OriginURL, + } + } + version, err := db.DmsApiClient.CreateVersion(ctx, deploymentID, versionID, dms.CreateVersionRequest{ + CliVersion: build.GetInfo().Version, + VersionType: versionType, + PreviousVersionId: previousVersionID, + Operations: staged, + GitInfo: gitInfo, + }) + if err != nil { + return fmt.Errorf("failed to create deployment version: %w", err) + } + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", deploymentID, version.VersionId) + + db.DmsAsyncOperationClient = dms.StartOperationBuffer(ctx, db.DmsApiClient, deploymentID, versionID) + return nil +} + +// drainOperationsAndCompleteVersion drains the buffered operations and completes the version. +// It returns whether the version completed successfully, which a destroy uses to decide whether +// to delete the deployment. Idempotent: the first call clears the buffer, so a deferred +// safety-net after an explicit completion is a no-op returning false. Also false when no version +// was created. +func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, success bool) (bool, error) { + db := &b.DeploymentBundle + buf := db.DmsAsyncOperationClient + if buf == nil { + return false, nil + } + db.DmsAsyncOperationClient = nil + + // Drain first; a recording error fails the deploy even when the resources applied. Its error + // is left to whoever drained (apply), not returned twice. + if buf.Drain() != nil { + success = false + } + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + deploymentID, versionID := deploymentAndNextVersion(b) + if err := db.DmsApiClient.CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { + return false, err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) + + return success, nil +} + +// logDeploymentVersion logs the deployment version URL. Workspace ID is omitted +// so the page stays clickable in a terminal and redirects correctly without it. +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, deploymentID string, version int64) { + if version == 0 { + return + } + + baseURL, err := url.Parse(b.WorkspaceClient(ctx).Config.CanonicalHostName()) + if err != nil { + // Only the link is lost, so report the version without it rather than failing + // a deploy over it. + log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) + cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", deploymentID, version)) + return + } + + cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) +} + +// deploymentMetadata describes the bundle this deploy came from and where it +// landed, mirroring what bundle/deploy/metadata computes for the metadata file. +func deploymentMetadata(b *bundle.Bundle) dms.Metadata { + p := dms.Metadata{ + DisplayName: b.Config.Bundle.Name, + TargetName: b.Config.Bundle.Target, + Mode: deploymentModeToSDK(b.Config.Bundle.Mode), + } + + ws := &bundledeployments.WorkspaceInfo{ + RootPath: b.Config.Workspace.RootPath, + FilePath: b.Config.Workspace.FilePath, + } + // In a source-linked deployment files are not copied, so resources read them + // from the sync root instead of file_path (see bundle/deploy/metadata.Compute). + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + ws.FilePath = b.SyncRootPath + ws.SourceLinked = true + } + // Only a deploy from a Databricks Git folder has one; a local worktree does not. + // bundle_root_path is relative to it, so the service requires both or neither. + if b.WorktreeRoot != nil && strings.HasPrefix(b.WorktreeRoot.Native(), "/Workspace/") { + ws.GitFolderPath = b.WorktreeRoot.Native() + ws.BundleRootPath = b.Config.Bundle.Git.BundleRootPath + } + p.Workspace = ws + return p +} + +// deploymentModeToSDK maps the bundle target's mode to the DMS enum. An unset mode +// maps to empty, which the service reads as "not reported". +func deploymentModeToSDK(mode config.Mode) bundledeployments.DeploymentMode { + switch mode { + case config.Development: + return bundledeployments.DeploymentModeDeploymentModeDevelopment + case config.Production: + return bundledeployments.DeploymentModeDeploymentModeProduction + default: + return "" + } +} diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go new file mode 100644 index 00000000000..f6854999db9 --- /dev/null +++ b/bundle/phases/dms_test.go @@ -0,0 +1,81 @@ +package phases + +import ( + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStagedOperationsCoversEveryTouchedResource(t *testing.T) { + // The version fixes its operation set, so anything the apply will write has to appear + // here. Keys go out in the service's form, without the CLI's "resources." prefix. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.foo": {Action: deployplan.Create}, + "resources.pipelines.bar": {Action: deployplan.Recreate}, + "resources.schemas.baz": {Action: deployplan.Delete}, + "resources.clusters.small": {Action: deployplan.Resize}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.ElementsMatch(t, []dms.StagedOperation{ + {ResourceKey: "resources.jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {ResourceKey: "resources.pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {ResourceKey: "resources.schemas.baz", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, + {ResourceKey: "resources.clusters.small", ActionType: bundledeployments.OperationActionTypeOperationActionTypeResize}, + }, staged) +} + +func TestStagedOperationsLeavesOutUntouchedResources(t *testing.T) { + // A skipped resource is never applied, so staging it would leave an operation pending for + // the life of the version. Undefined is not a real action either. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.touched": {Action: deployplan.Update}, + "resources.jobs.unchanged": {Action: deployplan.Skip}, + "resources.jobs.unknown": {Action: deployplan.Undefined}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.Equal(t, []dms.StagedOperation{ + {ResourceKey: "resources.jobs.touched", ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + }, staged) +} + +func TestStagedOperationsEmptyPlan(t *testing.T) { + staged, err := stagedOperations(&deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{}}) + require.NoError(t, err) + assert.Empty(t, staged) +} + +func TestActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := actionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Nothing is applied for these, so stagedOperations leaves them out rather than + // mapping them; the guard is here in case a new action type arrives without a mapping. + _, err := actionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = actionToSDK(deployplan.Undefined) + assert.Error(t, err) +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 2942dd4d05e..61428c02a21 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,6 +185,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY ("true" permits it) + // Rejects experimental.record_deployment_history: the feature is not usable yet. + validate.ValidateRecordDeploymentHistory(), + // Reject configured job_runs.idempotency_token; the CLI sets it on run-now. validate.ValidateJobRunIdempotencyToken(), diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 7b88290930c..6099cebdd14 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -172,7 +172,8 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s }() var planBundle direct.DeploymentBundle - if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false); err != nil { + // This plan check does not record deployment history, so no DMS client or deployment id. + if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, nil, ""); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 086ec1d600a..83612d37c16 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 6d938c5e03d..f0b3072c01c 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index fa5471e8f99..0092a417f01 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,13 +4,17 @@ import ( "context" "errors" "fmt" + "path" "path/filepath" + "strconv" "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" @@ -21,11 +25,15 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" "golang.org/x/mod/semver" ) @@ -189,6 +197,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, nil, err } + // The current deployment read from the service (nil, id "" if there is none yet). Used for the + // metadata diff and to reject a saved plan that predates the deployment's recorded version. + var dmsDeployment *bundledeployments.Deployment + var dmsDeploymentID string + shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" if shouldReadState { @@ -232,11 +245,66 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + + var dmsClient *dms.Client + if b.RecordsDeploymentHistory(ctx) { + deploymentID, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + + dmsClient, err = dms.NewClient(b.WorkspaceClient(ctx)) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + dmsDeploymentID = deploymentID + dmsDeployment = deployment + b.DeploymentBundle.DmsApiClient = dmsClient + + // Stamp the deployment and the version this run records onto every job and pipeline so + // the plan carries them. version_id is always known (last recorded + 1); deployment_id + // does not exist until a first deploy creates it, so it is left off here and the deploy + // phase stamps the created id. + lastVersionID := "" + if deployment != nil { + lastVersionID = deployment.LastVersionId + } + nextVersion, verr := dms.NextVersion(lastVersionID) + if verr != nil { + logdiag.LogError(ctx, verr) + return b, stateDesc, root.ErrAlreadyPrinted + } + muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} + if deploymentID != "" { + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ + DeploymentID: deploymentID, + LatestVersionID: deployment.LastVersionId, + } + }) + muts = append(muts, metadata.AnnotateDeployment(deploymentID)) + } + bundle.ApplySeqContext(ctx, b, muts...) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient, dmsDeploymentID); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + // The service holds this deployment, so it has to keep being recorded: deploying + // without recording would leave it describing resources that have moved on. + if !b.RecordsDeploymentHistory(ctx) && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { + logdiag.LogError(ctx, errors.New(`unsetting experimental.record_deployment_history is not supported + +This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`)) + return b, stateDesc, root.ErrAlreadyPrinted + } + // Warn when the state was last written by a newer CLI than the one // running now. The state schema version is a hard gate (dstate.Open // rejects a too-new state_version), but a state can be written by a @@ -277,6 +345,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } } + } var plan *deployplan.Plan @@ -300,8 +369,27 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle log.Warnf(ctx, "Plan was created with CLI version %s but current version is %s", plan.CLIVersion, currentVersion) } - // Validate that the plan's lineage and serial match the current state - // This must happen before any file operations + // The plan records the DMS deployment and version it targeted. Reject it if the live + // deployment moved on - a newer version (someone deployed since, possibly from another + // machine) or a different id (deleted and recreated) - so a stale plan is never applied on + // top of newer state. This runs before the lineage/serial check and is the authoritative + // stale guard for recorded bundles, whose local state is only a tombstone (its serial does + // not catch a deploy from elsewhere). Both sides are empty when not recording, a no-op there. + remoteLastVersion := "" + if dmsDeployment != nil { + remoteLastVersion = dmsDeployment.LastVersionId + } + if plan.LastVersionId != remoteLastVersion { + logdiag.LogError(ctx, fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion)) + return b, stateDesc, root.ErrAlreadyPrinted + } + if plan.DeploymentId != dmsDeploymentID { + logdiag.LogError(ctx, errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again")) + return b, stateDesc, root.ErrAlreadyPrinted + } + + // Validate that the plan's lineage and serial match the local state. This is the stale guard + // for non-recorded bundles (recorded ones are covered by the version check above). err = direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) if err != nil { logdiag.LogError(ctx, err) @@ -373,7 +461,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } t3 := time.Now() - phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, requiredEngine, libs, plan) + phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, requiredEngine, libs, plan, dmsDeployment) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Deploy", Value: time.Since(t3).Milliseconds(), @@ -427,6 +515,27 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS return engine.EngineSetting{Type: engine.Default, Source: engine.SourceDefault, IsDefault: true}, nil } +// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json +func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { + nodePath := path.Join(statePath, dms.DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil, nil + } + if err != nil { + return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + deploymentID := strconv.FormatInt(obj.ObjectId, 10) + deployment, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: dms.DeploymentName(deploymentID), + }) + if err != nil { + return "", nil, err + } + return deploymentID, deployment, nil +} + // isNewerVersion reports whether the state's recorded CLI version is strictly // newer than the running build. Both are bare versions without a leading "v". // An empty stateVersion (state not written by any CLI yet) or an unparseable diff --git a/libs/dms/client.go b/libs/dms/client.go new file mode 100644 index 00000000000..8dbf9915399 --- /dev/null +++ b/libs/dms/client.go @@ -0,0 +1,250 @@ +package dms + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// DeploymentNodeName is the workspace node DMS creates per deployment. Must +// match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +const DeploymentNodeName = "resources.deployment.json" + +// statePrefix is what a bundle state key carries and a DMS resource key does not: state calls a +// job "resources.jobs.foo", DMS calls it "jobs.foo". Every exported name here takes the state +// form; the prefix comes off where a request is built, and back on where a resource is read. +const statePrefix = "resources." + +// Client carries the calls the CLI makes to DMS, as methods below. Each one goes out through one +// of two halves: the generated client for the calls it can express, and hand-written requests +// for the two it cannot. +// +// TODO: Remove this and replace with the SDK. +type Client struct { + // Service is the generated client. + Service bundledeployments.BundleDeploymentsInterface + + // raw sends what the generated client cannot; see requester. + raw requester +} + +// NewClient returns a Client for the workspace w. +func NewClient(w *databricks.WorkspaceClient) (*Client, error) { + api, err := client.New(w.Config) + if err != nil { + return nil, err + } + return &Client{Service: w.BundleDeployments, raw: &rawClient{client: api}}, nil +} + +// DeploymentName and versionName are the two resource-name formats the service uses. Every +// call builds its name here, so a caller only ever passes ids. +func DeploymentName(deploymentID string) string { + return "deployments/" + deploymentID +} + +func versionName(deploymentID string, version int64) string { + return fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version) +} + +// CreateDeployment registers a deployment under parentPath and returns the id the server +// assigned it, which is the id of the workspace node it creates there. +func (c *Client) CreateDeployment(ctx context.Context, parentPath string, metadata Metadata) (string, error) { + dep := metadata.deployment() + dep.InitialParentPath = parentPath + + created, err := c.Service.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) + if err != nil { + return "", err + } + return deploymentIDFromName(created.Name) +} + +// UpdateDeployment writes the fields mask names onto the deployment. The service ignores every +// other field, so the mask is what decides the write. +func (c *Client) UpdateDeployment(ctx context.Context, deploymentID string, metadata Metadata, mask string) error { + return c.raw.UpdateDeployment(ctx, deploymentID, metadata.deployment(), mask) +} + +// DeleteDeployment removes the deployment record, which a completed destroy does. +func (c *Client) DeleteDeployment(ctx context.Context, deploymentID string) error { + return c.Service.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: DeploymentName(deploymentID), + }) +} + +// CreateVersion claims the version and stages the operations body carries. +func (c *Client) CreateVersion(ctx context.Context, deploymentID string, version int64, body CreateVersionRequest) (*bundledeployments.Version, error) { + return c.raw.CreateVersion(ctx, deploymentID, strconv.FormatInt(version, 10), body) +} + +// CompleteVersion closes the version out, which is what stops the service expiring its lease. +func (c *Client) CompleteVersion(ctx context.Context, deploymentID string, version int64, reason bundledeployments.VersionComplete) error { + _, err := c.Service.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName(deploymentID, version), + CompletionReason: reason, + }) + return err +} + +// UpdateOperation fills in one operation the version staged, and returns the sequence id the +// next update for that resource must send. +func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { + return c.raw.UpdateOperation(ctx, deploymentID, version, stateKey, sequenceID, update) +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, DeploymentName("")) + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + +// requester sends the two requests the generated client cannot express, so a test can capture +// what the CLI puts on the wire. Both are TODO(DMS): drop them once the spec catches up. +type requester interface { + // CreateVersion is hand-written because the generated struct has no operations: the field + // is at DEVELOPMENT stage, which keeps it out of the SDK until it is promoted. + CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) + + // UpdateDeployment is hand-written because the generated client has no such call yet. + UpdateDeployment(ctx context.Context, deploymentID string, deployment bundledeployments.Deployment, mask string) error + + // UpdateOperation is hand-written because the SDK types sequence_id as an int64 while + // the service sends a JSON string, so it cannot read the response. sequenceID is the + // token the previous update for this resource returned, or 0 for the first, which is + // what staging leaves. + UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (next string, err error) +} + +// CreateVersionRequest is the CreateVersion request body. +type CreateVersionRequest struct { + CliVersion string `json:"cli_version"` + VersionType VersionType `json:"version_type"` + // PreviousVersionId is the deployment's most recent version, unset for a + // deployment's first version. + PreviousVersionId string `json:"previous_version_id,omitempty"` + // GitInfo records where this version's source came from. The rest of the provenance - + // display name, target, mode, workspace paths - belongs to the deployment. + GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` + // Operations is every resource this version will touch; see StagedOperation. It sits in this + // body with the version's own fields because the request binds body: "version", and is input + // only - the response never carries it back. + Operations []StagedOperation `json:"operations,omitempty"` +} + +// StagedOperation is one resource the version will record an operation for. The service +// creates it in OPERATION_STATUS_PENDING at sequence id 0, and the CLI fills in the outcome +// with UpdateOperation as the resource is applied. +type StagedOperation struct { + // ResourceKey is the bundle state key; the request carries the form the service uses. + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` +} + +// operationResponse is the part of an operation response the CLI reads back. +type operationResponse struct { + // SequenceId is the concurrency token for the next update, typed as the service sends it. + SequenceId string `json:"sequence_id,omitempty"` +} + +// rawClient sends the requests the generated client cannot express. +type rawClient struct { + client *client.DatabricksClient +} + +func (r *rawClient) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { + staged := make([]StagedOperation, len(body.Operations)) + for i, op := range body.Operations { + op.ResourceKey = strings.TrimPrefix(op.ResourceKey, statePrefix) + staged[i] = op + } + body.Operations = staged + + var version bundledeployments.Version + path := "/api/2.0/bundle/" + DeploymentName(deploymentID) + "/versions" + err := r.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"version_id": versionID}, + body, &version) + if err != nil { + return nil, err + } + return &version, nil +} + +// newDeploymentUpdate builds the request body for update, holding exactly the masked fields for +// the same reason newUpdateRequest does. A map, not the SDK struct, whose omitempty tags would +// drop a masked field that is empty - which is how deployment_mode is cleared when a target stops +// setting mode. +func newDeploymentUpdate(deployment bundledeployments.Deployment, mask string) map[string]any { + values := map[string]any{ + "display_name": deployment.DisplayName, + "target_name": deployment.TargetName, + "deployment_mode": deployment.DeploymentMode, + "workspace_info": deployment.WorkspaceInfo, + } + + body := map[string]any{} + for field := range strings.SplitSeq(mask, ",") { + body[field] = values[field] + } + return body +} + +func (r *rawClient) UpdateDeployment(ctx context.Context, deploymentID string, deployment bundledeployments.Deployment, mask string) error { + path := "/api/2.0/bundle/" + DeploymentName(deploymentID) + return r.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"update_mask": mask}, + newDeploymentUpdate(deployment, mask), nil) +} + +// newUpdateRequest builds the request body for update. A field is in the body when the mask +// names it and absent otherwise, which is what the service requires: it rejects an update +// whose mask names a field the body leaves out, and an empty value is how a field is cleared - +// no state means the resource is gone, no error_message means an earlier failure is resolved. +// A map, not a struct, so presence cannot drift from the mask through an omitempty tag. +func newUpdateRequest(update OperationUpdate, sequenceID string) map[string]any { + body := map[string]any{"sequence_id": sequenceID} + // A masked state with no value is how the service is told the resource is gone, so a nil + // state is left out rather than sent empty. + if update.Fields.Has(FieldState) && update.State != nil { + body["state"] = string(update.State) + } + if update.Fields.Has(FieldResourceID) { + body["resource_id"] = update.ResourceID + } + if update.Fields.Has(FieldErrorMessage) { + body["error_message"] = update.ErrorMessage + } + if update.Fields.Has(FieldStatus) { + body["status"] = update.Status + } + return body +} + +func (r *rawClient) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { + body := newUpdateRequest(update, sequenceID) + + var result operationResponse + path := "/api/2.0/bundle/" + versionName(deploymentID, version) + "/operations/" + strings.TrimPrefix(stateKey, statePrefix) + err := r.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"update_mask": update.Fields.Mask()}, + body, &result) + if err != nil { + return "", err + } + return result.SequenceId, nil +} diff --git a/libs/dms/client_test.go b/libs/dms/client_test.go new file mode 100644 index 00000000000..867655bca4e --- /dev/null +++ b/libs/dms/client_test.go @@ -0,0 +1,67 @@ +package dms + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClientNamesEveryResourceTheSameWay(t *testing.T) { + // One format each, so a call only ever passes ids. + assert.Equal(t, "deployments/dep-1", DeploymentName("dep-1")) + assert.Equal(t, "deployments/dep-1/versions/2", versionName("dep-1", 2)) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} + +func TestUpdateRequestSendsExactlyTheMaskedFields(t *testing.T) { + // The service rejects an update whose mask names a field the body leaves out, and treats a + // field it does carry as written - so the body has to hold every masked field and nothing + // else, empty values included. Both masks the CLI builds are asserted on the wire by + // acceptance/bundle/dms; this pins the rule they both rely on. + update := OperationUpdate{ + Fields: DescribesResource, + State: json.RawMessage(`{"state":{"name":"foo"}}`), + ResourceID: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + } + + // A successful write reports no error, and the empty value is what clears an earlier one. + assert.Equal(t, map[string]any{ + "state": `{"state":{"name":"foo"}}`, + "resource_id": "job-1", + "error_message": "", + "status": bundledeployments.OperationStatusOperationStatusSucceeded, + "sequence_id": "3", + }, newUpdateRequest(update, "3")) + + // A failure keeps the recorded state, so state is absent rather than empty: naming it + // would clear what the resource last recorded. + assert.Equal(t, map[string]any{ + "error_message": "boom", + "status": bundledeployments.OperationStatusOperationStatusFailed, + "sequence_id": "3", + }, newUpdateRequest(NewFailureUpdate("job-1", nil, errors.New("boom")), "3")) + + // The deployment's own fields follow the same rule. Clearing deployment_mode - a target that + // stops setting mode - sends it empty, which the SDK struct's omitempty would have dropped. + deployment := Metadata{DisplayName: "b", TargetName: "t"}.deployment() + assert.Equal(t, map[string]any{ + "target_name": "t", + "deployment_mode": bundledeployments.DeploymentMode(""), + }, newDeploymentUpdate(deployment, "target_name,deployment_mode")) +} diff --git a/libs/dms/fields.go b/libs/dms/fields.go new file mode 100644 index 00000000000..05d39b387d7 --- /dev/null +++ b/libs/dms/fields.go @@ -0,0 +1,59 @@ +package dms + +import "strings" + +// Fields is the set of operation fields an update writes, sent as its update mask. The +// service rejects any other path, so this is the whole vocabulary. +type Fields uint8 + +const ( + FieldState Fields = 1 << iota + FieldErrorMessage + FieldResourceID + FieldStatus +) + +// DescribesResource is what a write that says how the resource looks claims: every field +// an update may change. +const DescribesResource = FieldState | FieldErrorMessage | FieldResourceID | FieldStatus + +// ClearsState is what a write that leaves no resource claims: a delete, and the delete half of +// a recreate. State is named so the service reads the absent value as a clear, which is what +// drops the resource from the deployment. It cannot name resource_id either: the service counts +// only a state it was given as recording one, so an id alongside a cleared state is refused. +const ClearsState = FieldState | FieldErrorMessage | FieldStatus + +// KeepsState is what a failure claims: mark it failed and leave state alone. State means +// the resource is as it was written; no state means a delete went through and nothing +// replaced it, so the resource really is gone and the deployment should say so. +// +// It cannot name resource_id: the service requires state in any mask that names the id, and +// naming state here would overwrite what the resource last recorded. +const KeepsState = FieldErrorMessage | FieldStatus + +// wireNames pairs each field with its name on the wire, in the order a mask lists them. +var wireNames = []struct { + field Fields + name string +}{ + {FieldState, "state"}, + {FieldErrorMessage, "error_message"}, + {FieldResourceID, "resource_id"}, + {FieldStatus, "status"}, +} + +// Has reports whether f contains every field in other. +func (f Fields) Has(other Fields) bool { + return f&other == other +} + +// Mask renders f as the update_mask the service expects, always in the same order. +func (f Fields) Mask() string { + names := make([]string, 0, len(wireNames)) + for _, w := range wireNames { + if f.Has(w.field) { + names = append(names, w.name) + } + } + return strings.Join(names, ",") +} diff --git a/libs/dms/fields_test.go b/libs/dms/fields_test.go new file mode 100644 index 00000000000..198a55efca8 --- /dev/null +++ b/libs/dms/fields_test.go @@ -0,0 +1,23 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFieldsMask(t *testing.T) { + // The order is fixed, so the same set always sends the same mask. + assert.Equal(t, "state,error_message,resource_id,status", DescribesResource.Mask()) + assert.Equal(t, "error_message,status", KeepsState.Mask()) + assert.Equal(t, "state", FieldState.Mask()) + assert.Empty(t, Fields(0).Mask()) +} + +func TestFieldsHas(t *testing.T) { + assert.True(t, DescribesResource.Has(FieldState)) + assert.False(t, KeepsState.Has(FieldState)) + // Has asks for every field, not any of them. + assert.True(t, DescribesResource.Has(FieldState|FieldStatus)) + assert.False(t, KeepsState.Has(FieldState|FieldStatus)) +} diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go new file mode 100644 index 00000000000..d63154a4790 --- /dev/null +++ b/libs/dms/metadata.go @@ -0,0 +1,95 @@ +package dms + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// VersionType identifies the kind of deployment a version records. +type VersionType = bundledeployments.VersionType + +const ( + VersionTypeDeploy VersionType = bundledeployments.VersionTypeVersionTypeDeploy + VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy +) + +// Metadata is what a version records about the bundle, its source and where it +// landed. The service copies these onto the deployment, so they describe it as of +// its most recent version. +type Metadata struct { + // DisplayName is the bundle's name, which the deployment is listed under. + DisplayName string + // TargetName is the bundle target that was deployed. + TargetName string + // Mode is the bundle target's mode, empty when the target sets none. + Mode bundledeployments.DeploymentMode + Workspace *bundledeployments.WorkspaceInfo +} + +// deploymentFields are the deployment's own metadata, in the order a mask lists them. Git is not +// among them: the service derives the deployment's from the version that carried it. +var deploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} + +// sameWorkspaceInfo compares the paths alone. The SDK records which fields a response carried in +// ForceSendFields, so a record read back never deep-equals one built here, and comparing the +// structs whole would report every run as a change. +func sameWorkspaceInfo(want, current *bundledeployments.WorkspaceInfo) bool { + if want == nil || current == nil { + return want == nil && current == nil + } + + a, b := *want, *current + a.ForceSendFields, b.ForceSendFields = nil, nil + return reflect.DeepEqual(a, b) +} + +// deployment renders the metadata the deployment owns. +func (m Metadata) deployment() bundledeployments.Deployment { + return bundledeployments.Deployment{ + DisplayName: m.DisplayName, + TargetName: m.TargetName, + DeploymentMode: m.Mode, + WorkspaceInfo: m.Workspace, + } +} + +// StaleFields returns the mask that brings current up to m, empty when the deployment already +// says what this run would say. current is nil before the first recorded deploy. +func (m Metadata) StaleFields(current *bundledeployments.Deployment) string { + if current == nil { + return strings.Join(deploymentFields, ",") + } + + want := m.deployment() + var stale []string + if want.DisplayName != current.DisplayName { + stale = append(stale, "display_name") + } + if want.TargetName != current.TargetName { + stale = append(stale, "target_name") + } + if want.DeploymentMode != current.DeploymentMode { + stale = append(stale, "deployment_mode") + } + if !sameWorkspaceInfo(want.WorkspaceInfo, current.WorkspaceInfo) { + stale = append(stale, "workspace_info") + } + return strings.Join(stale, ",") +} + +// NextVersion is the version number this run will create, given the deployment's most recent +// one. lastVersionID is empty before the deployment has any version. +func NextVersion(lastVersionID string) (int64, error) { + if lastVersionID == "" { + return 1, nil + } + last, err := strconv.ParseInt(lastVersionID, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse last_version_id %q: %w", lastVersionID, err) + } + return last + 1, nil +} diff --git a/libs/dms/metadata_test.go b/libs/dms/metadata_test.go new file mode 100644 index 00000000000..e3446466407 --- /dev/null +++ b/libs/dms/metadata_test.go @@ -0,0 +1,25 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNextVersion(t *testing.T) { + for _, tt := range []struct { + last string + want int64 + }{ + {"", 1}, + {"4", 5}, + } { + got, err := NextVersion(tt.last) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + + _, err := NextVersion("not-a-number") + assert.ErrorContains(t, err, "last_version_id") +} diff --git a/libs/dms/operation.go b/libs/dms/operation.go new file mode 100644 index 00000000000..38113853460 --- /dev/null +++ b/libs/dms/operation.go @@ -0,0 +1,129 @@ +package dms + +import ( + "encoding/json" + "fmt" + "unicode/utf8" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// maxStateSize is the largest serialized state DMS accepts per operation. More than this +// and the resource cannot be recorded at all, so the deploy fails rather than leaving the +// service holding a resource with no state. +const maxStateSize = 64 * 1024 + +// maxErrorMessageSize is how much of a failure's message the service stores. +const maxErrorMessageSize = 16 * 1024 + +// StatusPending is what the service calls an operation it has recorded but not applied, which +// is where a recreate sits between deleting the old resource and creating the new one. Not in +// the SDK: the enum ships server-first while it is staged DEVELOPMENT, so the generated client +// carries only the terminal two. +const StatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" + +// OperationUpdate is one write to an operation the version staged: the fields it claims +// and their values. It is built where the outcome is known, so a malformed one fails the +// resource that produced it rather than the upload at the end of apply. +type OperationUpdate struct { + // Fields is the mask to send. It is taken literally: a field named here is written, + // one left out keeps the value it had. + Fields Fields + + // State is the serialized state after the operation, and nil for a delete. + State json.RawMessage + + ResourceID string + Status bundledeployments.OperationStatus + ErrorMessage string +} + +// NewStateUpdate describes how the resource looks now: state is the serialized envelope the +// state DB just persisted, and nil for a delete. inProgress marks a write that is half of a +// larger change - a recreate's delete - so an interrupted deploy does not report it finished. +func NewStateUpdate(resourceID string, state json.RawMessage, inProgress bool) (OperationUpdate, error) { + if len(state) > maxStateSize { + return OperationUpdate{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxStateSize) + } + + status := bundledeployments.OperationStatusOperationStatusSucceeded + if inProgress { + status = StatusPending + } + + // No state means nothing is left to describe, so the write clears it rather than recording + // it as empty: an empty value is a state the service refuses on a delete that succeeded, + // while an absent one is how the resource stops being listed at all. + if state == nil { + return OperationUpdate{ + Fields: ClearsState, + Status: status, + }, nil + } + + return OperationUpdate{ + Fields: DescribesResource, + State: state, + ResourceID: resourceID, + Status: status, + }, nil +} + +// NewFailureUpdate records that an operation did not apply, so the history says why a resource +// failed rather than leaving it pending. state is what the resource still has, which the service +// requires of a failure on a live one; nil when nothing is left to describe. +func NewFailureUpdate(resourceID string, state json.RawMessage, cause error) OperationUpdate { + // Summarized, not cause.Error(): for an API failure that adds the status and error + // code, which is often the most actionable part of the history. + message := diag.FormatAPIErrorSummary(cause) + if len(message) > maxErrorMessageSize { + message = message[:maxErrorMessageSize] + // The cut can land inside a rune, and the service stores a string. Drop the partial + // one: at most UTFMax-1 bytes of it can be left, so a message that was already + // invalid loses those bytes rather than being stripped away entirely. + for range utf8.UTFMax - 1 { + if utf8.ValidString(message) { + break + } + message = message[:len(message)-1] + } + } + + fields := KeepsState + if state != nil { + // The id travels with the state, which the service requires of any mask naming one. + fields = DescribesResource + } + + return OperationUpdate{ + Fields: fields, + State: state, + ResourceID: resourceID, + Status: bundledeployments.OperationStatusOperationStatusFailed, + ErrorMessage: message, + } +} + +// Merge folds a later update into u, for a resource written twice before either upload ran. +// Each field comes from whichever update claimed it, newer winning when both did, and the +// mask is the union. What an update claims is decided where it is built, not here. +func (u OperationUpdate) Merge(newer OperationUpdate) OperationUpdate { + merged := u + merged.Fields = u.Fields | newer.Fields + + if newer.Fields.Has(FieldState) { + merged.State = newer.State + } + if newer.Fields.Has(FieldResourceID) { + merged.ResourceID = newer.ResourceID + } + if newer.Fields.Has(FieldErrorMessage) { + merged.ErrorMessage = newer.ErrorMessage + } + if newer.Fields.Has(FieldStatus) { + merged.Status = newer.Status + } + + return merged +} diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go new file mode 100644 index 00000000000..bc58ff5285e --- /dev/null +++ b/libs/dms/operation_buffer.go @@ -0,0 +1,171 @@ +package dms + +import ( + "context" + "encoding/json" + "fmt" + "sync" +) + +// bufferedOperations caps how far ahead of the service a deploy may get; DMS is what the next +// plan reads. +const bufferedOperations = 10 + +// stagedSequenceID is what version creation leaves on a staged operation, so a resource's first +// update sends it as the precondition. +const stagedSequenceID = "0" + +// OperationBuffer records each state write with DMS for one deployment version, off the apply +// path: writes are queued and sent on one background goroutine. It exists only while a bundle +// records deployment history; callers hold a nil buffer otherwise and must not call it. +type OperationBuffer struct { + client *Client + deploymentID string + versionNum int64 + + // The buffer. queue holds bundle state keys, and pending the newest update per key, so a + // second write for a resource replaces the first. + queue chan string + done chan struct{} + stopQueue func() + + // sequenceIDs holds the token the last update for a resource returned. A resource absent + // from it has only what staging left, so its first update sends that. Unguarded: run is the + // only goroutine that writes, one update at a time. + sequenceIDs map[string]string + + // mu guards the fields below. + mu sync.Mutex + + // pending holds the newest update per resource key, absent once the writer takes it. + pending map[string]OperationUpdate + + err error +} + +// StartOperationBuffer opens the buffer for the version the caller just created. The version +// must already exist: operations record under it, and nothing here creates it. +func StartOperationBuffer(ctx context.Context, client *Client, deploymentID string, versionNum int64) *OperationBuffer { + b := &OperationBuffer{ + client: client, + deploymentID: deploymentID, + versionNum: versionNum, + queue: make(chan string, bufferedOperations), + done: make(chan struct{}), + pending: make(map[string]OperationUpdate), + sequenceIDs: make(map[string]string), + } + b.stopQueue = sync.OnceFunc(func() { close(b.queue) }) + go b.run(ctx) + return b +} + +// RecordOperation records a state write. state is the serialized envelope, nil for a delete. +// An earlier failure does not stop it: keep recording, best effort. +func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey string, inProgress bool, resourceID string, state json.RawMessage) { + update, err := NewStateUpdate(resourceID, state, inProgress) + if err != nil { + b.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) + return + } + + b.record(resourceKey, update) +} + +// RecordFailure records that a resource did not apply, so the history says why rather +// than leaving the resource out. +func (b *OperationBuffer) RecordFailure(resourceKey, resourceID string, state json.RawMessage, cause error) { + b.record(resourceKey, NewFailureUpdate(resourceID, state, cause)) +} + +// record makes update the one waiting for resourceKey, waiting itself while the queue is +// full. Recording after Drain panics, so every caller must return before Drain. +func (b *OperationBuffer) record(resourceKey string, update OperationUpdate) { + b.mu.Lock() + waiting, queued := b.pending[resourceKey] + if queued { + update = waiting.Merge(update) + } + b.pending[resourceKey] = update + b.mu.Unlock() + + // when already queued: the writer reads the map when it gets to the key, + // so it picks up what was just stored. + // + // Only when the resource is not already queued, enqueue it. + if !queued { + b.queue <- resourceKey + } +} + +// take claims the update waiting for resourceKey. +func (b *OperationBuffer) take(resourceKey string) (OperationUpdate, bool) { + b.mu.Lock() + defer b.mu.Unlock() + + update, ok := b.pending[resourceKey] + delete(b.pending, resourceKey) + return update, ok +} + +func (b *OperationBuffer) run(ctx context.Context) { + defer close(b.done) + + for resourceKey := range b.queue { + update, ok := b.take(resourceKey) + if !ok { + // Unreachable: a key is queued only when nothing was waiting for it. Guard so + // a stray key could never write a zero-valued update. + continue + } + + // Keep going after a failure, so one bad write does not drop everything behind it. + if err := b.write(ctx, resourceKey, update); err != nil { + b.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) + } + } +} + +// write sends one update, at the sequence id the resource is at. +func (b *OperationBuffer) write(ctx context.Context, key string, update OperationUpdate) error { + sequenceID, written := b.sequenceIDs[key] + if !written { + sequenceID = stagedSequenceID + } + + next, err := b.client.UpdateOperation(ctx, b.deploymentID, b.versionNum, key, sequenceID, update) + if err != nil { + return err + } + + // The next write for this resource echoes the sequence id this one earned. + b.sequenceIDs[key] = next + + return nil +} + +// Drain waits for everything buffered to reach the service and returns the first write error, +// which fails the deploy: DMS is the source of truth for what exists. Safe to call twice. +func (b *OperationBuffer) Drain() error { + b.stopQueue() + <-b.done + return b.Err() +} + +// setErr keeps the first error; one failure is enough to fail the deploy. +func (b *OperationBuffer) setErr(err error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.err == nil { + b.err = err + } +} + +// Err returns the first recording error, or nil. +func (b *OperationBuffer) Err() error { + b.mu.Lock() + defer b.mu.Unlock() + + return b.err +} diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go new file mode 100644 index 00000000000..a57f25c7de5 --- /dev/null +++ b/libs/dms/operation_buffer_test.go @@ -0,0 +1,78 @@ +package dms + +import ( + "encoding/json" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What the service ends up holding - the version a deploy claims, the operations it stages, the +// completion it reports - is asserted end to end by acceptance/bundle/dms. What is left here is +// what a deploy cannot reach: the queue's coalescing and the size limit. No test here goes +// through the API - the queue is driven directly, so nothing waits on a request. + +// queued builds a buffer whose queue nothing drains, so a test drives record and take itself. +func queued() *OperationBuffer { + return &OperationBuffer{ + queue: make(chan string, bufferedOperations), + pending: make(map[string]OperationUpdate), + } +} + +func stateUpdate(t *testing.T, name string) OperationUpdate { + t.Helper() + update, err := NewStateUpdate("id-1", json.RawMessage(`{"state":{"name":"`+name+`"}}`), false) + require.NoError(t, err) + return update +} + +func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { + s := queued() + + // Two writes for one resource with nothing draining. They carry the resource's full + // state, so only the newest needs to go: one slot in the queue, not two. + s.record("resources.jobs.foo", stateUpdate(t, "v1")) + s.record("resources.jobs.foo", stateUpdate(t, "v2")) + + assert.Len(t, s.queue, 1) + update, ok := s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v2"}}`, string(update.State)) + + // Taken means a request has it, and an in-flight request cannot be recalled, so the next + // write gets its own slot rather than joining it. + s.record("resources.jobs.foo", stateUpdate(t, "v3")) + assert.Len(t, s.queue, 2) + update, ok = s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v3"}}`, string(update.State)) +} + +func TestBufferFailsOnOversizedState(t *testing.T) { + // The service will not take a state this large, so the resource cannot be recorded. + // Failing here says so, where reporting nothing would leave DMS without the resource + // and the next plan would create it again. + s := queued() + + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", json.RawMessage(strings.Repeat("x", maxStateSize+1))) + + assert.ErrorContains(t, s.Err(), "exceeds the 65536 byte limit") + assert.Empty(t, s.queue) + assert.Empty(t, s.pending) +} + +func TestDrainIsIdempotent(t *testing.T) { + // Nothing recorded, so no transport is needed: this is the second drain, which must not + // panic on an already closed queue. + s := queued() + s.done = make(chan struct{}) + s.stopQueue = sync.OnceFunc(func() { close(s.queue) }) + go s.run(t.Context()) + + require.NoError(t, s.Drain()) + require.NoError(t, s.Drain()) +} diff --git a/libs/dms/operation_test.go b/libs/dms/operation_test.go new file mode 100644 index 00000000000..2bae9a81144 --- /dev/null +++ b/libs/dms/operation_test.go @@ -0,0 +1,83 @@ +package dms + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "unicode/utf8" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What these updates put on the wire - the mask, the status, the state a write carries and a +// failure leaves alone - is asserted by acceptance/bundle/dms. What is left here are the limits +// and the merge, which a deploy cannot reach. + +func TestNewFailureUpdateTruncatesLongError(t *testing.T) { + // Truncated rather than rejected: a message over the limit would make recording + // fail and hide the error it is reporting. + update := NewFailureUpdate("job-123", nil, errors.New(strings.Repeat("x", maxErrorMessageSize+100))) + + assert.Len(t, update.ErrorMessage, maxErrorMessageSize) +} + +func TestNewFailureUpdatePreservesUTF8OnTruncation(t *testing.T) { + // The cut lands one byte into the emoji, so a byte-wise truncation would leave a partial + // rune behind and the service stores state and messages as strings. + msg := strings.Repeat("a", maxErrorMessageSize-1) + "❌" + "x" + + update := NewFailureUpdate("job-123", nil, errors.New(msg)) + + assert.True(t, utf8.ValidString(update.ErrorMessage)) + // The whole emoji went, so the message is shorter than the limit rather than exactly it. + assert.Equal(t, strings.Repeat("a", maxErrorMessageSize-1), update.ErrorMessage) +} + +func TestMergeLetsAWriteSupersedeAFailure(t *testing.T) { + // A failure is not the last word. A retry that writes state wins whole - state, id and + // mask - and the mask names error_message so the recorded failure is cleared. The service + // rejects a succeeded operation that still carries an error. + failed := NewFailureUpdate("id-old", nil, errors.New("boom")) + retried, err := NewStateUpdate("id-new", json.RawMessage(`{"state":{"name":"after"}}`), false) + require.NoError(t, err) + + merged := failed.Merge(retried) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, merged.Status) + assert.Empty(t, merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.JSONEq(t, `{"state":{"name":"after"}}`, string(merged.State)) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeKeepsTheWritesStateAndMask(t *testing.T) { + // A failure claims only status and error_message, so the write's state, id and mask + // survive: the resource stays listed as it was written, now marked failed. + write, err := NewStateUpdate("id-new", json.RawMessage(`{"state":{"name":"before"}}`), false) + require.NoError(t, err) + failed := NewFailureUpdate("id-old", nil, errors.New("boom")) + + merged := write.Merge(failed) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, merged.Status) + assert.Equal(t, "boom", merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.JSONEq(t, `{"state":{"name":"before"}}`, string(merged.State)) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeLetsADeleteClearTheState(t *testing.T) { + // A delete legitimately carries no state, and merging must let it through: the resource + // is gone, and keeping the state it replaces would leave it listed. + write, err := NewStateUpdate("id-1", json.RawMessage(`{"state":{"name":"before"}}`), false) + require.NoError(t, err) + deleted, err := NewStateUpdate("id-1", nil, false) + require.NoError(t, err) + + merged := write.Merge(deleted) + + assert.Nil(t, merged.State) +} diff --git a/libs/dms/resources.go b/libs/dms/resources.go new file mode 100644 index 00000000000..565571415f6 --- /dev/null +++ b/libs/dms/resources.go @@ -0,0 +1,41 @@ +package dms + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Resource is what DMS holds for one resource of a deployment, as of the last operation +// that recorded it. +type Resource struct { + // Key is the bundle state key, as everything outside this package spells it. + Key string + ID string + + // State is the state the last operation recorded, as the opaque string the service + // stores, and empty when no operation recorded one. + State string +} + +// ListResources returns every resource DMS holds for the deployment. +func (c *Client) ListResources(ctx context.Context, deploymentID string) ([]Resource, error) { + it := c.Service.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: DeploymentName(deploymentID), + }) + + var out []Resource + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + out = append(out, Resource{ + Key: statePrefix + res.ResourceKey, + ID: res.ResourceId, + State: res.State, + }) + } + return out, nil +} diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go new file mode 100644 index 00000000000..3545036297a --- /dev/null +++ b/libs/testserver/bundledeployments.go @@ -0,0 +1,615 @@ +package testserver + +import ( + "bytes" + "encoding/json" + "fmt" + "path" + "slices" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. + +// maxOperationsPerVersion mirrors the service's compiled-in default. A bundle past it cannot +// be recorded, since the operation set is fixed when the version is created. +const maxOperationsPerVersion = 800 + +// operationStatusPending is what CreateVersion leaves on a staged operation. Declared here +// because the SDK enum is generated from the OpenAPI spec, which trails the service proto. +const operationStatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" + +// State is kept in FakeWorkspace.DmsDeployments, keyed by deployment ID. + +// dmsDeploymentNodeName is the workspace node name the service uses for deployments. +// It must match DEPLOYMENT_NODE_NAME on the service side (DeploymentWhsClient). +const dmsDeploymentNodeName = "resources.deployment.json" + +// dmsUpdatableOperationFields are the update_mask paths UpdateOperation accepts. Any +// other path is rejected, and action_type in particular is fixed when the operation is +// created. +var dmsUpdatableOperationFields = []string{"state", "error_message", "resource_id", "status"} + +// DmsDeployment holds a deployment record together with the versions and +// resources recorded under it, so the read APIs (ListVersions/ListResources) +// can serve back what deploys wrote. +type DmsDeployment struct { + Deployment bundledeployments.Deployment + Versions map[string]*bundledeployments.Version + // resources is the latest resource state per resource key, updated as + // operations are recorded. + Resources map[string]bundledeployments.Resource + // operations holds the recorded operations by resource name. The service keeps + // one per resource per version, so a resource written twice in a version updates + // its operation rather than adding another. + Operations map[string]*bundledeployments.Operation + // lastSuccessfulVersionID is the highest version completed successfully. + // The read path treats a non-empty value as "DMS owns the state"; + // the SDK Deployment struct does not yet carry this field. + LastSuccessfulVersionID string +} + +func (s *FakeWorkspace) CreateDeployment(req Request) Response { + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + if dep.InitialParentPath == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "initial_parent_path is required"}, + } + } + + defer s.LockUnlock()() + + // The service registers the deployment as a workspace node under + // initial_parent_path and uses that node's ID as the deployment ID, so a + // get-status on the node path is how clients look the deployment back up. + nodePath := path.Join(dep.InitialParentPath, dmsDeploymentNodeName) + if resp, ok := s.requireParentDirectory(nodePath); !ok { + return resp + } + objectID := nextID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + // The service registers its own node type, which get-status reports back; + // a plain FILE is what the fake used to claim and the real API never returns. + ObjectType: "BUNDLE_DEPLOYMENT", + Path: nodePath, + ObjectId: objectID, + }, + } + + // The record carries no version yet; last_version_id stays empty until + // the first CreateVersion. A failed registration leaves a record with no versions. + deploymentID := strconv.FormatInt(objectID, 10) + s.DmsDeploymentNodes[deploymentID] = nodePath + + if resp, ok := checkWorkspaceInfo(dep.WorkspaceInfo); !ok { + return resp + } + + dep.Name = "deployments/" + deploymentID + dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.DmsDeployments[deploymentID] = &DmsDeployment{ + Deployment: dep, + Versions: map[string]*bundledeployments.Version{}, + Resources: map[string]bundledeployments.Resource{}, + Operations: map[string]*bundledeployments.Operation{}, + } + return Response{Body: dep} +} + +func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +// deploymentBody renders a deployment with last_successful_version_id, which +// the SDK struct doesn't yet carry. Embedding in a wrapper won't work because +// Deployment.MarshalJSON silently drops sibling fields. +func deploymentBody(d *DmsDeployment) (map[string]any, error) { + raw, err := json.Marshal(d.Deployment) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + if d.LastSuccessfulVersionID != "" { + body["last_successful_version_id"] = d.LastSuccessfulVersionID + } + return body, nil +} + +// dmsUpdatableDeploymentFields are the update_mask paths UpdateDeployment accepts. +var dmsUpdatableDeploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} + +// checkWorkspaceInfo rejects a bundle_root_path without the git_folder_path it is relative to, +// which is what the service does. +func checkWorkspaceInfo(ws *bundledeployments.WorkspaceInfo) (Response, bool) { + if ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { + return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together"), false + } + return Response{}, true +} + +func (s *FakeWorkspace) UpdateDeployment(req Request, deploymentID string) Response { + var raw map[string]json.RawMessage + if err := json.Unmarshal(req.Body, &raw); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + update := map[string]bool{} + for path := range strings.SplitSeq(updateMask, ",") { + path = strings.TrimSpace(path) + if !slices.Contains(dmsUpdatableDeploymentFields, path) { + return dmsInvalidArgument("update_mask path " + path + " is not updatable") + } + if _, sent := raw[path]; !sent { + // An empty value is how a field is cleared, so the field has to be there to say so. + return dmsInvalidArgument(path + " is required when '" + path + "' is in update_mask (an empty value clears it)") + } + update[path] = true + } + + if resp, ok := checkWorkspaceInfo(dep.WorkspaceInfo); !ok { + return resp + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Only the masked paths are written; every other field of the body is ignored. + if update["display_name"] { + d.Deployment.DisplayName = dep.DisplayName + } + if update["target_name"] { + d.Deployment.TargetName = dep.TargetName + } + if update["deployment_mode"] { + d.Deployment.DeploymentMode = dep.DeploymentMode + } + if update["workspace_info"] { + d.Deployment.WorkspaceInfo = dep.WorkspaceInfo + } + + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + // The service trashes the deployment's workspace node, so a later get-status + // on the node path reports the deployment as absent. + if nodePath, ok := s.DmsDeploymentNodes[deploymentID]; ok { + delete(s.files, nodePath) + } + delete(s.DmsDeploymentNodes, deploymentID) + delete(s.DmsDeployments, deploymentID) + return Response{Body: map[string]any{}} +} + +func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response { + versionID := req.URL.Query().Get("version_id") + + var version bundledeployments.Version + if err := json.Unmarshal(req.Body, &version); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + // operations rides in the version body like every other field, but is absent from the + // generated struct while the field is at DEVELOPMENT stage, so it is read separately. + // Embedding Version instead would promote its own UnmarshalJSON and silently drop this + // field. It is input only, so it is read here and never returned. + var staged struct { + Operations []struct { + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` + } `json:"operations"` + } + if err := json.Unmarshal(req.Body, &staged); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // version_id must be numerically greater than the most recent version, + // and previous_version_id must name that version to detect concurrent deploys. + next, err := strconv.ParseInt(versionID, 10, 64) + if err != nil || next < 1 { + return dmsInvalidArgument("version_id must be a positive integer, got " + versionID) + } + var last int64 + if d.Deployment.LastVersionId != "" { + last, _ = strconv.ParseInt(d.Deployment.LastVersionId, 10, 64) + } + if next <= last { + return dmsInvalidArgument("version_id " + versionID + " must be greater than the most recent version " + d.Deployment.LastVersionId) + } + if version.PreviousVersionId != d.Deployment.LastVersionId { + return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.Deployment.LastVersionId) + } + + // Note: deployment lock not modelled. Tests kill the CLI mid-apply, leaving + // the version in-progress forever, whereas the real service lets the lease expire. + + // A version records its whole operation set up front; there is no API to add one later. + if len(staged.Operations) > maxOperationsPerVersion { + return Response{ + StatusCode: 429, + Body: map[string]string{ + "error_code": "RESOURCE_EXHAUSTED", + "message": fmt.Sprintf("a version may stage at most %d operations, got %d", maxOperationsPerVersion, len(staged.Operations)), + }, + } + } + seen := make(map[string]bool, len(staged.Operations)) + for _, staged := range staged.Operations { + switch { + case staged.ResourceKey == "": + return dmsInvalidArgument("operations.resource_key is required") + case !strings.Contains(staged.ResourceKey, "."): + return dmsInvalidArgument("operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): " + staged.ResourceKey) + case staged.ActionType == "": + return dmsInvalidArgument("operations.action_type is required and must not be UNSPECIFIED for resource " + staged.ResourceKey) + case seen[staged.ResourceKey]: + return dmsInvalidArgument("operations must have distinct resource_keys; duplicate: " + staged.ResourceKey) + } + seen[staged.ResourceKey] = true + } + + d.Deployment.LastVersionId = versionID + version.Name = "deployments/" + deploymentID + "/versions/" + versionID + version.VersionId = versionID + version.Status = bundledeployments.VersionStatusVersionStatusInProgress + d.Versions[versionID] = &version + + // The deployment's git provenance is derived from the version that carried it; the rest of + // its metadata is written through CreateDeployment and UpdateDeployment. + d.Deployment.GitInfo = version.GitInfo + + // Each staged operation starts pending at sequence 0, and the CLI fills in its outcome + // with UpdateOperation as the resource is applied. + for _, staged := range staged.Operations { + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + staged.ResourceKey + d.Operations[opName] = &bundledeployments.Operation{ + Name: opName, + ResourceKey: staged.ResourceKey, + ActionType: staged.ActionType, + Status: operationStatusPending, + SequenceId: 0, + } + } + + return Response{Body: version} +} + +func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID string) Response { + var completeReq bundledeployments.CompleteVersionRequest + if err := json.Unmarshal(req.Body, &completeReq); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + v, ok := d.Versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + v.Status = bundledeployments.VersionStatusVersionStatusCompleted + v.CompletionReason = completeReq.CompletionReason + if completeReq.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + d.LastSuccessfulVersionID = versionID + } + return Response{Body: *v} +} + +func (s *FakeWorkspace) Heartbeat() Response { + return Response{Body: bundledeployments.HeartbeatResponse{}} +} + +// operationBody renders an operation the way the service does: sequence_id as a +// JSON string, which the SDK struct cannot express (it types the field int64). +func operationBody(op *bundledeployments.Operation) (map[string]any, error) { + raw, err := json.Marshal(op) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + body["sequence_id"] = strconv.FormatInt(op.SequenceId, 10) + return body, nil +} + +// UpdateOperation applies a later write for a resource already recorded in this +// version. sequence_id is the concurrency precondition and increments on success. +func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, resourceKey string) Response { + // sequence_id arrives as a string, which the SDK struct cannot hold (it types the + // field int64), so read the body twice: once for the typed fields with that key + // removed, and once for the precondition alone. + var raw map[string]json.RawMessage + if err := json.Unmarshal(req.Body, &raw); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var precondition struct { + SequenceId string `json:"sequence_id"` + } + if err := json.Unmarshal(req.Body, &precondition); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + delete(raw, "sequence_id") + typedBody, err := json.Marshal(raw) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + var op bundledeployments.Operation + if err := json.Unmarshal(typedBody, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + // Only masked paths are written; other fields keep their values. + // Omitting state keeps the already-recorded state. + update := map[string]bool{} + for path := range strings.SplitSeq(updateMask, ",") { + path = strings.TrimSpace(path) + if !slices.Contains(dmsUpdatableOperationFields, path) { + return dmsInvalidArgument("update_mask path " + path + " is not updatable") + } + // state is the exception: masked with no value is how the service is told the resource + // is gone, which is what stops it being listed. + if _, sent := raw[path]; !sent && path != "state" { + // An empty value is how a field is cleared, so the field has to be there to say so. + return dmsInvalidArgument(path + " is required when '" + path + "' is in update_mask (an empty value clears it)") + } + update[path] = true + } + + // The service parses the status as a proto enum, so a value outside the enum arrives as + // UNSPECIFIED and is refused. Checked here too: a status the API does not have otherwise + // passes every local test and fails only against a real workspace. + if update["status"] { + switch op.Status { + case bundledeployments.OperationStatusOperationStatusSucceeded, + bundledeployments.OperationStatusOperationStatusFailed, + operationStatusPending: + default: + return dmsInvalidArgument("status is required and must not be UNSPECIFIED when 'status' is in update_mask") + } + } + + // The service counts only a state it was given as recording one, so these ask whether a + // value arrived rather than whether the mask names the field. + _, stateSent := raw["state"] + recordsState := update["state"] && stateSent + + // A resource_id can be written but not cleared, and only alongside the state it belongs to. + if update["resource_id"] { + if op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required when 'resource_id' is in update_mask") + } + if !recordsState { + return dmsInvalidArgument("state must be in update_mask when 'resource_id' is") + } + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + existing, ok := d.Operations[opName] + if !ok { + return dmsNotFound("operation " + opName) + } + if precondition.SequenceId != strconv.FormatInt(existing.SequenceId, 10) { + return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) + } + + // Invariants check the operation after the update, not the request. + // The mask leaves unspecified fields unchanged. + after := *existing + if update["state"] { + after.State = op.State + } + if update["error_message"] { + after.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + after.ResourceId = op.ResourceId + } + if update["status"] { + after.Status = op.Status + } + + failed := after.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && after.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + + // A failure may leave no state only where nothing is left to describe: a create that never + // landed, or a recreate whose delete already went through. + if failed && after.State == "" && leavesLiveResource(after.ActionType) { + return dmsInvalidArgument("state is required for a " + string(after.ActionType) + + " operation, because it acts on a resource that already exists and cannot destroy it, even when it fails") + } + + // An operation with state must identify its resource via resource_id, + // even for failed operations reporting prior state. + if after.State != "" && after.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for an operation that records state") + } + + // Only the masked fields change; action_type and resource_key stay as created. + if update["state"] { + existing.State = op.State + } + if update["error_message"] { + existing.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + existing.ResourceId = op.ResourceId + } + if update["status"] { + existing.Status = op.Status + } + existing.SequenceId++ + + body, err := operationBody(existing) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + + // Only an update that names state moves the resource: naming it with no value clears + // it and removes the resource, and an update that leaves it out - a failure reporting + // its outcome - must not disturb what the deployment already holds. Every version + // stages its operations without state, so re-deriving this from the operation + // regardless of the mask would drop a resource whose deploy failed before writing. + if update["state"] { + if existing.State == "" { + delete(d.Resources, resourceKey) + } else { + d.Resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: existing.ResourceId, + ResourceType: existing.ResourceType, + LastActionType: existing.ActionType, + LastVersionId: versionID, + State: existing.State, + } + } + } else if resource, projected := d.Resources[resourceKey]; projected && update["resource_id"] { + // resource_id is mirrored too, for a resource the deployment still holds. + resource.ResourceId = existing.ResourceId + d.Resources[resourceKey] = resource + } + + return Response{Body: body} +} + +func (s *FakeWorkspace) ListResources(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Sort by resource key so the response order is deterministic. + keys := make([]string, 0, len(d.Resources)) + for key := range d.Resources { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]bundledeployments.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, d.Resources[key]) + } + return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} +} + +// leavesLiveResource reports whether a failed operation of this type leaves a resource behind +// that it still has to describe. A create leaves nothing, and a recreate already deleted. +func leavesLiveResource(action bundledeployments.OperationActionType) bool { + switch action { + case bundledeployments.OperationActionTypeOperationActionTypeCreate, + bundledeployments.OperationActionTypeOperationActionTypeRecreate: + return false + default: + return true + } +} + +// dmsNotFound returns the RESOURCE_DOES_NOT_EXIST error shape the DMS API uses, +// which the SDK maps to apierr.ErrNotFound. +func dmsNotFound(what string) Response { + return Response{ + StatusCode: 404, + // Content-Type is required for the SDK to parse the body into a typed error, + // which is what callers match against apierr.ErrResourceDoesNotExist. + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": what + " does not exist", + }, + } +} + +// dmsAborted returns the 409 ABORTED error the server uses for the version +// optimistic-concurrency check. +func dmsInvalidArgument(message string) Response { + return Response{ + StatusCode: 400, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } +} + +func dmsAborted(message string) Response { + return Response{ + StatusCode: 409, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "ABORTED", "message": message}, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 2f077b1b631..b544c17538e 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -248,6 +248,15 @@ type FakeWorkspace struct { // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv + + // DmsDeployments holds Deployment Metadata Service (DMS) records, keyed by + // deployment ID. Each record carries its versions and latest resource state. + DmsDeployments map[string]*DmsDeployment + + // DmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment made for + // it. An ID appears here before DmsDeployments has a record, which its first version + // creates, so the node is what makes the ID valid in between. + DmsDeploymentNodes map[string]string } func (s *FakeWorkspace) LockUnlock() func() { @@ -496,6 +505,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, + DmsDeployments: map[string]*DmsDeployment{}, + DmsDeploymentNodes: map[string]string{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae7..33a742cc2ce 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -279,6 +279,35 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.JobsCreate(req) }) + // Deployment Metadata Service (DMS) endpoints. + server.Handle("POST", "/api/2.0/bundle/deployments", func(req Request) any { + return req.Workspace.CreateDeployment(req) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.GetDeployment(req.Vars["deployment_id"]) + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.UpdateDeployment(req, req.Vars["deployment_id"]) + }) + server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { + return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { + return req.Workspace.Heartbeat() + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}", func(req Request) any { + return req.Workspace.UpdateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"], req.Vars["resource_key"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { + return req.Workspace.ListResources(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob if err := json.Unmarshal(req.Body, &request); err != nil { diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 61c3f271468..0b6a80fd71b 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "strings" ) @@ -71,6 +72,22 @@ func ResourceTypes() []string { return names } +// DeploymentURL returns the workspace URL for a bundle deployment: +// /deployments/?version=. Version pins the page to the deploy that produced it. +func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { + if deploymentID == "" { + return "" + } + + baseURL.Path = "deployments/" + deploymentID + if version > 0 { + values := baseURL.Query() + values.Set("version", strconv.FormatInt(version, 10)) + baseURL.RawQuery = values.Encode() + } + return baseURL.String() +} + // JobRunPath returns the modern workspace path for a job run, of the form // // jobs//runs/ diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index a398167e027..b92df037c84 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -141,6 +141,55 @@ func TestResourceURL(t *testing.T) { } } +func TestDeploymentURL(t *testing.T) { + tests := []struct { + name string + deploymentID string + version int64 + base url.URL + expected string + }{ + { + name: "id and version", + deploymentID: "996980114684409", + version: 2, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409?version=2", + }, + { + // The version is only known once CreateVersion has run, so link to the + // deployment itself rather than emitting version=0. + name: "zero version omits the query", + deploymentID: "996980114684409", + version: 0, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409", + }, + { + // A base URL carrying ?w= keeps it, so a vanity or legacy + // host still addresses the right workspace. + name: "preserves an existing query", + deploymentID: "42", + version: 7, + base: url.URL{Scheme: "https", Host: "host.com", RawQuery: "w=123"}, + expected: "https://host.com/deployments/42?version=7&w=123", + }, + { + name: "empty id returns empty", + deploymentID: "", + version: 1, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, DeploymentURL(tt.base, tt.deploymentID, tt.version)) + }) + } +} + func TestHasWorkspaceIDInHostname(t *testing.T) { tests := []struct { name string