bundle: record and read deployment state via DMS

acceptance/bundle/dms/existing-state/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: dms-existing-state
+
+experimental:
+  record_deployment_history: false
+
+resources:
+  jobs:
+    one:
+      name: one

acceptance/bundle/dms/existing-state/output.txt

@@ -0,0 +1,45 @@
+
+=== Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+
+=== Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time
+>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true
+
+>>> musterr [CLI] bundle deploy
+Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again
+
+
+=== No deployment was created in DMS
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+
+=== Destroy clears the tracked resources, so recording can be enabled afterwards
+>>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.one
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default
+
+Deleting files...
+Destroy complete!
+
+>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"target_name": "default"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}}

acceptance/bundle/dms/existing-state/script

@@ -0,0 +1,17 @@
+title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --oneline
+
+title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time"
+trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true"
+trace musterr $CLI bundle deploy
+
+title "No deployment was created in DMS"
+trace print_requests.py //api/2.0/bundle --sort --oneline
+
+title "Destroy clears the tracked resources, so recording can be enabled afterwards"
+trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false"
+trace $CLI bundle destroy --auto-approve
+trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --oneline

acceptance/bundle/dms/multiple-resources/databricks.yml

@@ -0,0 +1,18 @@
+bundle:
+  name: dms-multiple-resources
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    one:
+      name: one
+    two:
+      name: two
+    three:
+      name: three
+    four:
+      name: four
+    five:
+      name: five

acceptance/bundle/dms/multiple-resources/output.txt

@@ -0,0 +1,25 @@
+
+=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it.
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //versions/1/operations --sort --del-body state --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}}
+
+=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}

acceptance/bundle/dms/multiple-resources/script

@@ -0,0 +1,7 @@
+title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it."
+trace $CLI bundle deploy
+trace print_requests.py //versions/1/operations --sort --del-body state --oneline
+
+title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --oneline

acceptance/bundle/dms/no-resources/databricks.yml

@@ -0,0 +1,5 @@
+bundle:
+  name: dms-no-resources
+
+experimental:
+  record_deployment_history: true

acceptance/bundle/dms/no-resources/output.txt

@@ -0,0 +1,78 @@
+
+=== First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --get
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+>>> jq .deployment_id .databricks/bundle/default/resources.json
+"[UUID]"
+
+=== Redeploy: the persisted ID is reused, so no second deployment is created
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --get
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]/resources"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}

acceptance/bundle/dms/no-resources/script

@@ -0,0 +1,8 @@
+title "First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --get
+trace jq .deployment_id .databricks/bundle/default/resources.json
+
+title "Redeploy: the persisted ID is reused, so no second deployment is created"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --get

acceptance/bundle/dms/record/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: dms-record
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo

acceptance/bundle/dms/record/output.txt

@@ -0,0 +1,140 @@
+
+=== Deploy: the server assigns the deployment ID, and a version + create operation are recorded
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "jobs.foo"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[NUMID]",
+    "resource_key": "jobs.foo",
+    "state": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "foo",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+
+=== The server-assigned deployment ID is persisted in the local state file
+>>> jq .deployment_id .databricks/bundle/default/resources.json
+"[UUID]"
+
+=== Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== Destroy: a destroy version and delete operation are recorded, then the deployment is deleted
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_requests.py //api/2.0/bundle --sort
+{
+  "method": "DELETE",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "3"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DESTROY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/operations",
+  "q": {
+    "resource_key": "jobs.foo"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_DELETE",
+    "resource_key": "jobs.foo",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}