diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index dee38fa687..61b0119eb5 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -5,8 +5,7 @@ ### Bundles * Added support for UC external locations (direct mode only) ([#4484](https://github.com/databricks/cli/pull/4484)) * Log artifact build output in debug mode ([#4208](https://github.com/databricks/cli/pull/4208)) +* Fix bundle init not working in Azure Government ([#4286](https://github.com/databricks/cli/pull/4286)) +* engine/direct: Replace server_side_default with more precise backend_default rule in bundle plan ([#4490](https://github.com/databricks/cli/pull/4490)) ### Dependency updates - -### Bundles -* Fix bundle init not working in Azure Government ([#4286](https://github.com/databricks/cli/pull/4286)) diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index ab374309c4..86957a974c 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -172,7 +172,7 @@ }, "tasks[task_key='ServerlessTestTask'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='ServerlessTestTask'].timeout_seconds": { @@ -199,7 +199,7 @@ }, "tasks[task_key='TestTask'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='TestTask'].timeout_seconds": { diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.direct.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.direct.txt index 75d0788301..c5e00b49cb 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.direct.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.direct.txt @@ -8,6 +8,7 @@ Changes detected: ~ edition: null -> "ADVANCED" ~ libraries: [{"glob":{"include":"/Workspace/Users/someuser@databricks.com/lakeflow_pipeline/transformations/**"}},{"glob":{"include":"/Workspace/Users/foo@databricks.com/another/**"}}] -> [{"notebook":{"path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files/nb"}}] ~ name: "lakeflow-pipeline-[UNIQUE_NAME]" -> "test-pipeline-[UNIQUE_NAME]" + ~ root_path: "/Workspace/Users/someuser@databricks.com/lakeflow_pipeline" -> null ~ storage: "/Shared/old_storage" -> "/Shared/new_storage" Error: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt index 385516aec5..7b85803529 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt @@ -9,6 +9,7 @@ Changes detected: ~ edition: null -> "ADVANCED" ~ libraries: [{"glob":{"include":"/Workspace/Users/[USERNAME]/lakeflow_pipeline/transformations/**"}},{"glob":{"include":"/Workspace/Users/foo@databricks.com/another/**"}}] -> [{"notebook":{"path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb"}}] ~ name: "lakeflow-pipeline" -> "test-pipeline" + ~ root_path: "/Workspace/Users/[USERNAME]/lakeflow_pipeline" -> null Error: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index a5d6967b19..6d11ed7b1c 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -39,12 +39,10 @@ cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null # Any failures after this point will be considered as "bug detected" by fuzzer. echo INPUT_CONFIG_OK -$CLI bundle plan -o json &> plan.json -cat plan.json | contains.py '!panic' '!internal error' > /dev/null - # Check both text and JSON plan for no changes # Note, expect that there maybe more than one resource unchanged -$CLI bundle plan | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan +$CLI bundle plan -o json &> LOG.planjson +cat LOG.planjson | contains.py '!panic' '!internal error' > /dev/null +verify_no_drift.py LOG.planjson -$CLI bundle plan -o json > plan.json -verify_no_drift.py plan.json +$CLI bundle plan | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index 5c7bb6e043..c249c7b850 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -80,12 +80,12 @@ }, "tasks[task_key='main'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { @@ -202,7 +202,7 @@ }, "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" }, "tags['myjob_name']": { @@ -237,7 +237,7 @@ "changes": { "storage_location": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } } diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json index 15e6c7379a..67b82cbabb 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json @@ -242,12 +242,12 @@ }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -268,7 +268,7 @@ }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -283,7 +283,7 @@ }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { @@ -342,7 +342,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json index 527a30cf67..5f228bec81 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json @@ -242,12 +242,12 @@ }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -268,7 +268,7 @@ }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -283,7 +283,7 @@ }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { @@ -342,7 +342,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 539235fab3..dcdd47e087 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -92,12 +92,12 @@ Building python_artifact... }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -118,7 +118,7 @@ Building python_artifact... }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -133,7 +133,7 @@ Building python_artifact... }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { @@ -155,7 +155,7 @@ Building python_artifact... { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } @@ -211,12 +211,12 @@ Building python_artifact... }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -237,7 +237,7 @@ Building python_artifact... }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -252,7 +252,7 @@ Building python_artifact... }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { @@ -274,7 +274,7 @@ Building python_artifact... { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } diff --git a/acceptance/bundle/migrate/runas/out.plan.json b/acceptance/bundle/migrate/runas/out.plan.json index 92ab37022b..bdab0ae3bb 100644 --- a/acceptance/bundle/migrate/runas/out.plan.json +++ b/acceptance/bundle/migrate/runas/out.plan.json @@ -34,7 +34,7 @@ "changes": { "run_as": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "input_only", "old": { "service_principal_name": "[UUID]" }, diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json index 18585bffb5..3328c1be7c 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json @@ -25,7 +25,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } @@ -57,7 +57,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } @@ -89,7 +89,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } @@ -121,7 +121,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index 627f4ff3de..ea57340a31 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -44,7 +44,6 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "data_security_mode": "SINGLE_USER", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, "node_type_id": "[NODE_TYPE_ID]", @@ -61,20 +60,15 @@ }, "aws_attributes": { "action": "skip", - "reason": "server_side_default", + "reason": "managed", "remote": { "availability": "SPOT_WITH_FALLBACK", "zone_id": "us-east-1c" } }, - "data_security_mode": { - "action": "skip", - "reason": "server_side_default", - "remote": "SINGLE_USER" - }, "driver_node_type_id": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "[NODE_TYPE_ID]" }, "enable_elastic_disk": { @@ -123,7 +117,6 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "data_security_mode": "SINGLE_USER", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, "node_type_id": "[NODE_TYPE_ID]", @@ -144,20 +137,15 @@ }, "aws_attributes": { "action": "skip", - "reason": "server_side_default", + "reason": "managed", "remote": { "availability": "SPOT_WITH_FALLBACK", "zone_id": "us-east-1c" } }, - "data_security_mode": { - "action": "skip", - "reason": "server_side_default", - "remote": "SINGLE_USER" - }, "driver_node_type_id": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "[NODE_TYPE_ID]" }, "enable_elastic_disk": { @@ -201,7 +189,6 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "data_security_mode": "SINGLE_USER", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, "node_type_id": "[NODE_TYPE_ID]", @@ -225,20 +212,15 @@ }, "aws_attributes": { "action": "skip", - "reason": "server_side_default", + "reason": "managed", "remote": { "availability": "SPOT_WITH_FALLBACK", "zone_id": "us-east-1c" } }, - "data_security_mode": { - "action": "skip", - "reason": "server_side_default", - "remote": "SINGLE_USER" - }, "driver_node_type_id": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "[NODE_TYPE_ID]" }, "enable_elastic_disk": { @@ -279,7 +261,6 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "data_security_mode": "SINGLE_USER", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, "node_type_id": "[NODE_TYPE_ID]", @@ -301,20 +282,15 @@ }, "aws_attributes": { "action": "skip", - "reason": "server_side_default", + "reason": "managed", "remote": { "availability": "SPOT_WITH_FALLBACK", "zone_id": "us-east-1c" } }, - "data_security_mode": { - "action": "skip", - "reason": "server_side_default", - "remote": "SINGLE_USER" - }, "driver_node_type_id": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "[NODE_TYPE_ID]" }, "enable_elastic_disk": { diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt index 339aa0a105..78b00d25ee 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -95,7 +95,6 @@ Deployment complete! }, "cluster_id":"[CLUSTER_ID]", "cluster_name":"test-cluster-[UNIQUE_NAME]", - "data_security_mode":"SINGLE_USER", "driver_node_type_id":"[NODE_TYPE_ID]", "enable_elastic_disk":false, "node_type_id":"[NODE_TYPE_ID]", diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json index c96a75f91a..acf0cfa65b 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json @@ -47,7 +47,6 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "data_security_mode": "SINGLE_USER", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, "node_type_id": "[NODE_TYPE_ID]", @@ -60,20 +59,15 @@ "changes": { "aws_attributes": { "action": "skip", - "reason": "server_side_default", + "reason": "managed", "remote": { "availability": "SPOT_WITH_FALLBACK", "zone_id": "us-east-1c" } }, - "data_security_mode": { - "action": "skip", - "reason": "server_side_default", - "remote": "SINGLE_USER" - }, "driver_node_type_id": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "[NODE_TYPE_ID]" }, "enable_elastic_disk": { @@ -119,7 +113,6 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "data_security_mode": "SINGLE_USER", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, "node_type_id": "[NODE_TYPE_ID]", @@ -133,20 +126,15 @@ "changes": { "aws_attributes": { "action": "skip", - "reason": "server_side_default", + "reason": "managed", "remote": { "availability": "SPOT_WITH_FALLBACK", "zone_id": "us-east-1c" } }, - "data_security_mode": { - "action": "skip", - "reason": "server_side_default", - "remote": "SINGLE_USER" - }, "driver_node_type_id": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "[NODE_TYPE_ID]" }, "enable_elastic_disk": { @@ -193,7 +181,6 @@ }, "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", - "data_security_mode": "SINGLE_USER", "driver_node_type_id": "[NODE_TYPE_ID]", "enable_elastic_disk": false, "node_type_id": "[NODE_TYPE_ID]", @@ -207,20 +194,15 @@ "changes": { "aws_attributes": { "action": "skip", - "reason": "server_side_default", + "reason": "managed", "remote": { "availability": "SPOT_WITH_FALLBACK", "zone_id": "us-east-1c" } }, - "data_security_mode": { - "action": "skip", - "reason": "server_side_default", - "remote": "SINGLE_USER" - }, "driver_node_type_id": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "[NODE_TYPE_ID]" }, "enable_elastic_disk": { diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt index 6460372627..43c78fa780 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -51,7 +51,6 @@ Deployment complete! }, "cluster_id":"[CLUSTER_ID]", "cluster_name":"test-cluster-[UNIQUE_NAME]", - "data_security_mode":"SINGLE_USER", "driver_node_type_id":"[NODE_TYPE_ID]", "enable_elastic_disk":false, "node_type_id":"[NODE_TYPE_ID]", diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index b8f0621ae9..a6709a074f 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -39,7 +39,7 @@ "changes": { "storage_location": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } } diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json index f84c096a34..929dfdff92 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json @@ -147,7 +147,7 @@ }, "tasks[task_key='TestTask2'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='TestTask2'].timeout_seconds": { diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json index 00230b72bb..816bda2d67 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json @@ -11,12 +11,12 @@ }, "tasks[task_key='usage_logs_grants'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='usage_logs_grants'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='usage_logs_grants'].timeout_seconds": { diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json index bf51ae0831..15963bc007 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json @@ -98,7 +98,7 @@ }, "tasks[task_key='TestTask'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='TestTask'].timeout_seconds": { diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json index 61107b5a0c..448eafd5d3 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json @@ -80,7 +80,7 @@ }, "tasks[task_key='main_task'].new_cluster.data_security_mode": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "SINGLE_USER" }, "tasks[task_key='main_task'].new_cluster.enable_elastic_disk": { @@ -90,12 +90,12 @@ }, "tasks[task_key='main_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='main_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main_task'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json index 941b23e5c8..c5e456b5c0 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json index 12cb612f48..1914747039 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json index f818786292..6779ef9018 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index d8aa0e3c54..06e3eed8c7 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json index 82ecdbde2b..c15dd9a21c 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json index ba7bd1fa01..44682af8c0 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json index 2fd7740fa3..b46b11176f 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index ff650d6c00..0d192dd875 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json index 6fdb4e8387..20cbccb035 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json @@ -51,7 +51,7 @@ }, "tasks[task_key='main'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='main'].timeout_seconds": { diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json index a8a906dcc4..af3718c59f 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json @@ -25,7 +25,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[FOO_ID]" } } diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json index f9bd45988f..f11c8a77fc 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json @@ -25,7 +25,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[FOO_ID]" } } diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json index 4827d439b0..fad11f3014 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json @@ -25,7 +25,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[FOO_ID]" } } diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json index 34c36a5032..6fd7896b3d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json @@ -25,7 +25,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[FOO_ID]" } } diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index 7a86cca523..021901f34e 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -69,7 +69,7 @@ }, "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[MY_ID]" } } diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json index bb5aa94039..2f837c11f0 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json @@ -38,7 +38,7 @@ }, "warehouse_id": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "input_only", "old": "[TEST_DEFAULT_WAREHOUSE_ID]", "new": "[TEST_DEFAULT_WAREHOUSE_ID]" } diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json index a8b5135efa..c4705197b1 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json @@ -35,7 +35,7 @@ }, "warehouse_id": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "input_only", "old": "[TEST_DEFAULT_WAREHOUSE_ID]", "new": "[TEST_DEFAULT_WAREHOUSE_ID]" } diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json index c63af6ed43..8d70f68dc7 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json @@ -38,7 +38,7 @@ }, "warehouse_id": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "input_only", "old": "[TEST_DEFAULT_WAREHOUSE_ID]", "new": "[TEST_DEFAULT_WAREHOUSE_ID]" } diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json index 729c1cf466..8805337438 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json @@ -20,7 +20,7 @@ "changes": { "warehouse_id": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "input_only", "old": "[TEST_DEFAULT_WAREHOUSE_ID]", "new": "[TEST_DEFAULT_WAREHOUSE_ID]" } diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json index dec07013ea..f6bfbfbc03 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json @@ -39,7 +39,7 @@ }, "storage_location": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json index ab9cd13fe9..2c7d9da5a7 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json @@ -242,12 +242,12 @@ }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -268,7 +268,7 @@ }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -283,7 +283,7 @@ }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { @@ -342,7 +342,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json index fcb6c73404..d31f339a20 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json @@ -135,12 +135,12 @@ }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -155,7 +155,7 @@ }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -170,7 +170,7 @@ }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { @@ -243,7 +243,7 @@ "changes": { "storage": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "dbfs:/pipelines/[UUID]" } } diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json index 29887dea16..4c4d021866 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json @@ -124,12 +124,12 @@ }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -144,7 +144,7 @@ }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -159,7 +159,7 @@ }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json index 92a6cc7464..812094baeb 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json @@ -121,12 +121,12 @@ }, "tasks[task_key='notebook_task'].notebook_task.source": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "WORKSPACE" }, "tasks[task_key='notebook_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='notebook_task'].timeout_seconds": { @@ -141,7 +141,7 @@ }, "tasks[task_key='python_wheel_task'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='python_wheel_task'].timeout_seconds": { @@ -156,7 +156,7 @@ }, "tasks[task_key='refresh_pipeline'].run_if": { "action": "skip", - "reason": "server_side_default", + "reason": "backend_default", "remote": "ALL_SUCCESS" }, "tasks[task_key='refresh_pipeline'].timeout_seconds": { diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 481c5a4a93..0d8bddc2e9 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -145,9 +145,7 @@ func DetectChanges(ctx context.Context, b *bundle.Bundle, engine engine.EngineTy if entry.Changes != nil { for path, changeDesc := range entry.Changes { - // TODO: as for now in bundle plan all remote-side changes are considered as server-side defaults. - // Once it is solved - stop skipping server-side defaults in these checks and remove hardcoded default. - if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonServerSideDefault { + if changeDesc.Action == deployplan.Skip { continue } diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 7098e63383..e0dcd9b288 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -98,11 +98,11 @@ type ChangeDesc struct { // Possible values for Reason field const ( - ReasonServerSideDefault = "server_side_default" - ReasonAlias = "alias" - ReasonRemoteAlreadySet = "remote_already_set" - ReasonEmpty = "empty" - ReasonCustom = "custom" + ReasonBackendDefault = "backend_default" + ReasonAlias = "alias" + ReasonRemoteAlreadySet = "remote_already_set" + ReasonEmpty = "empty" + ReasonCustom = "custom" // Special reason that results in removing this change from the plan ReasonDrop = "!drop" diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 23cb3cd3ff..5c518adcc9 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -2,6 +2,7 @@ package direct import ( "context" + "encoding/json" "errors" "fmt" "maps" @@ -382,13 +383,12 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if reason, ok := shouldSkip(generatedCfg, path, ch); ok { ch.Action = deployplan.Skip ch.Reason = reason - } else if ch.New == nil && ch.Old == nil && ch.Remote != nil && path.IsDotString() { - // The field was not set by us, but comes from the remote state. - // This could either be server-side default or a policy. - // In any case, this is not a change we should react to. - // Note, we only consider struct fields here. Adding/removing elements to/from maps and slices should trigger updates. + } else if reason, ok := shouldSkipBackendDefault(cfg, path, ch); ok { ch.Action = deployplan.Skip - ch.Reason = deployplan.ReasonServerSideDefault + ch.Reason = reason + } else if reason, ok := shouldSkipBackendDefault(generatedCfg, path, ch); ok { + ch.Action = deployplan.Skip + ch.Reason = reason } else if action, reason := shouldUpdateOrRecreate(cfg, path); action != deployplan.Undefined { ch.Action = action ch.Reason = reason @@ -461,6 +461,43 @@ func shouldUpdateOrRecreate(cfg *dresources.ResourceLifecycleConfig, path *struc return deployplan.Undefined, "" } +// shouldSkipBackendDefault checks if a change should be skipped because the remote value +// is a known backend default. Applies when old and new are nil but remote is set. +// If the rule has allowed values, the remote value must match one of them. +func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, ch *deployplan.ChangeDesc) (string, bool) { + if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil { + return "", false + } + for _, rule := range cfg.BackendDefaults { + if !path.HasPatternPrefix(rule.Field) { + continue + } + if len(rule.Values) == 0 { + return deployplan.ReasonBackendDefault, true + } + if matchesAllowedValue(ch.Remote, rule.Values) { + return deployplan.ReasonBackendDefault, true + } + } + return "", false +} + +// matchesAllowedValue checks if the remote value matches one of the allowed JSON values. +// Each json.RawMessage is unmarshaled into the same type as remote for comparison. +func matchesAllowedValue(remote any, values []json.RawMessage) bool { + remoteType := reflect.TypeOf(remote) + for _, raw := range values { + candidate := reflect.New(remoteType).Interface() + if err := json.Unmarshal(raw, candidate); err != nil { + continue + } + if structdiff.IsEqual(remote, reflect.ValueOf(candidate).Elem().Interface()) { + return true + } + } + return false +} + func allEmpty(values ...any) bool { for _, v := range values { if v == nil { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 8210b1ed32..559685b891 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -870,6 +870,9 @@ func validateResourceConfig(t *testing.T, stateType reflect.Type, cfg *ResourceL for _, p := range cfg.IgnoreRemoteChanges { assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "IgnoreRemoteChanges: %s", p.Field) } + for _, p := range cfg.BackendDefaults { + assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "BackendDefaults: %s", p.Field) + } } func setupTestServerClient(t *testing.T) (*testserver.Server, *databricks.WorkspaceClient) { diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index eba9e62f5a..0b71f51382 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -2,6 +2,7 @@ package dresources import ( _ "embed" + "encoding/json" "sync" "github.com/databricks/cli/libs/structs/structpath" @@ -14,6 +15,36 @@ type FieldRule struct { Reason string `yaml:"reason"` } +// BackendDefaultRule represents a field that may be set by the backend as a default. +// When old and new are nil but remote is set, and the field matches, the change is skipped. +// If Values is non-empty, the remote value must match one of the allowed values. +type BackendDefaultRule struct { + Field *structpath.PatternNode `yaml:"field"` + Values []json.RawMessage `yaml:"values,omitempty"` +} + +// UnmarshalYAML implements custom YAML unmarshaling for BackendDefaultRule. +// Values are parsed from native YAML types and stored as JSON bytes. +func (b *BackendDefaultRule) UnmarshalYAML(unmarshal func(any) error) error { + type helper struct { + Field *structpath.PatternNode `yaml:"field"` + Values []any `yaml:"values,omitempty"` + } + var h helper + if err := unmarshal(&h); err != nil { + return err + } + b.Field = h.Field + for _, v := range h.Values { + raw, err := json.Marshal(v) + if err != nil { + return err + } + b.Values = append(b.Values, json.RawMessage(raw)) + } + return nil +} + // ResourceLifecycleConfig defines lifecycle behavior for a resource type. type ResourceLifecycleConfig struct { // IgnoreRemoteChanges: field patterns where remote changes are ignored (output-only, policy-set). @@ -27,6 +58,10 @@ type ResourceLifecycleConfig struct { // UpdateIDOnChanges: field patterns that trigger UpdateWithID when changed. UpdateIDOnChanges []FieldRule `yaml:"update_id_on_changes,omitempty"` + + // BackendDefaults: fields where the backend may set defaults. + // When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped. + BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"` } // Config is the root configuration structure for resource lifecycle behavior. @@ -50,6 +85,7 @@ var ( IgnoreLocalChanges: nil, RecreateOnChanges: nil, UpdateIDOnChanges: nil, + BackendDefaults: nil, } ) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 08db4c47bf..4eb39aeb2e 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -6,6 +6,8 @@ # update_id_on_changes: fields that trigger UpdateWithID (ID may change) # ignore_remote_changes: fields where remote changes are ignored # ignore_local_changes: fields where local changes are ignored (can't be updated via API) +# backend_defaults: fields where the backend may set defaults (skipped when old/new are nil but remote is set) +# Optional "values" list constrains which remote values are allowed (as JSON-compatible literals). # # Each field entry has: # field: the field path @@ -15,14 +17,16 @@ resources: jobs: ignore_remote_changes: + # Same as clusters.{aws,azure,gcp}_attributes — see clusters/resource_cluster.go#L361-L363 + # s.SchemaPath("aws_attributes").SetSuppressDiff() + # s.SchemaPath("azure_attributes").SetSuppressDiff() + # s.SchemaPath("gcp_attributes").SetSuppressDiff() - field: tasks[*].new_cluster.aws_attributes - # These are not "output_only", because they can be set by the user as well. reason: managed - field: tasks[*].new_cluster.azure_attributes reason: managed - field: tasks[*].new_cluster.gcp_attributes reason: managed - - field: job_clusters[*].new_cluster.aws_attributes reason: managed - field: job_clusters[*].new_cluster.azure_attributes @@ -30,11 +34,68 @@ resources: - field: job_clusters[*].new_cluster.gcp_attributes reason: managed - # can also be added to backend_defaults with value=false once we have that + backend_defaults: + # Same as clusters.enable_elastic_disk — see clusters/resource_cluster.go#L331 + # s.SchemaPath("enable_elastic_disk").SetComputed() - field: tasks[*].new_cluster.enable_elastic_disk - reason: managed - field: job_clusters[*].new_cluster.enable_elastic_disk - reason: managed + + # Same as clusters.enable_local_disk_encryption — see clusters/resource_cluster.go#L332 + # s.SchemaPath("enable_local_disk_encryption").SetComputed() + - field: tasks[*].new_cluster.enable_local_disk_encryption + - field: job_clusters[*].new_cluster.enable_local_disk_encryption + + # Same as clusters.node_type_id — see clusters/resource_cluster.go#L333 + # s.SchemaPath("node_type_id").SetComputed() + - field: tasks[*].new_cluster.node_type_id + - field: job_clusters[*].new_cluster.node_type_id + + # Same as clusters.driver_node_type_id — see clusters/resource_cluster.go#L334 + # s.SchemaPath("driver_node_type_id").SetComputed() + - field: tasks[*].new_cluster.driver_node_type_id + - field: job_clusters[*].new_cluster.driver_node_type_id + + # Same as clusters.driver_instance_pool_id — see clusters/resource_cluster.go#L335 + # s.SchemaPath("driver_instance_pool_id").SetComputed() + - field: tasks[*].new_cluster.driver_instance_pool_id + - field: job_clusters[*].new_cluster.driver_instance_pool_id + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/jobs/resource_job.go#L531 + # s.SchemaPath("format").SetComputed() + - field: format + values: ["MULTI_TASK", "SINGLE_TASK"] + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/jobs/resource_job.go#L639 + # s.SchemaPath("task", "run_if").SetSuppressDiffWithDefault(jobs.RunIfAllSuccess) + - field: "tasks[*].run_if" + values: ["ALL_SUCCESS"] + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/jobs/resource_job.go#L640 + # s.SchemaPath("task", "for_each_task", "task", "run_if").SetSuppressDiffWithDefault(jobs.RunIfAllSuccess) + - field: "tasks[*].for_each_task.task.run_if" + values: ["ALL_SUCCESS"] + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/jobs/resource_job.go#L527 + # s.SchemaPath("run_as").SetComputed() + - field: run_as + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/jobs/resource_job.go#L521-L524 + # s.SchemaPath("task", "notebook_task", "source").SetSuppressDiff() + # s.SchemaPath("task", "spark_python_task", "source").SetSuppressDiff() + # s.SchemaPath("task", "sql_task", "file", "source").SetSuppressDiff() + # s.SchemaPath("task", "dbt_task", "source").SetSuppressDiff() + - field: tasks[*].notebook_task.source + - field: tasks[*].for_each_task.task.notebook_task.source + - field: tasks[*].spark_python_task.source + - field: tasks[*].for_each_task.task.spark_python_task.source + - field: tasks[*].sql_task.file.source + - field: tasks[*].for_each_task.task.sql_task.file.source + - field: tasks[*].dbt_task.source + - field: tasks[*].for_each_task.task.dbt_task.source + + # Same as clusters.data_security_mode: backend sets this when not specified + - field: tasks[*].new_cluster.data_security_mode + - field: job_clusters[*].new_cluster.data_security_mode pipelines: recreate_on_changes: @@ -56,6 +117,19 @@ resources: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L209 - field: ingestion_definition.ingest_from_uc_foreign_catalog reason: immutable + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L204 + - field: gateway_definition.connection_id + reason: immutable + - field: gateway_definition.connection_name + reason: immutable + - field: gateway_definition.gateway_storage_catalog + reason: immutable + - field: gateway_definition.gateway_storage_schema + reason: immutable + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L209 + - field: ingestion_definition.ingest_from_uc_foreign_catalog + reason: immutable + ignore_remote_changes: # "id" is handled in a special way before any fields changed # However, it is also part of RemotePipeline via CreatePipeline. @@ -63,12 +137,37 @@ resources: - field: id reason: "!drop" - field: run_as - reason: not_returned_by_api + reason: input_only + ignore_local_changes: # "id" is output-only, providing it in config would be a mistake - field: id reason: "!drop" + backend_defaults: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L238 + # s.SchemaPath("storage").SetCustomSuppressDiff(suppressStorageDiff) + # Backend generates storage path like dbfs:/pipelines/ when not set by user. + - field: storage + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L218 + # s.SchemaPath("cluster", "node_type_id").SetComputed() + - field: clusters[*].node_type_id + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L219 + # s.SchemaPath("cluster", "driver_node_type_id").SetComputed() + - field: clusters[*].driver_node_type_id + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L220 + # s.SchemaPath("cluster", "enable_local_disk_encryption").SetComputed() + - field: clusters[*].enable_local_disk_encryption + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/pipelines/resource_pipeline.go#L229-L230 + # s.SchemaPath("event_log", "catalog").SetComputed() + # s.SchemaPath("event_log", "schema").SetComputed() + - field: event_log.catalog + - field: event_log.schema + models: recreate_on_changes: # Recreate matches current behavior of Terraform. It is possible to rename without recreate @@ -94,6 +193,10 @@ resources: recreate_on_changes: - field: artifact_location reason: immutable + backend_defaults: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/mlflow/resource_mlflow_experiment.go#L34 + # SetForceNew().SetSuppressDiff(): backend generates artifact_location when not set by user + - field: artifact_location ignore_remote_changes: # Tags updates are not supported by TF. This mirrors that behaviour. - field: tags @@ -118,8 +221,42 @@ resources: reason: immutable - field: route_optimized reason: immutable + ignore_remote_changes: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L370 + # common.CustomizeSchemaPath(m, "config", "traffic_config").SetComputed() + # Routes have custom SuppressDiff (lines 387-388) + - field: config.traffic_config + reason: managed + backend_defaults: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 + # common.CustomizeSchemaPath(m, "config", "served_entities", "name").SetComputed() + - field: config.served_entities[*].name + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L384 + # common.CustomizeSchemaPath(m, "config", "served_entities", "workload_type").SetComputed() + - field: config.served_entities[*].workload_type + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L372 + # common.CustomizeSchemaPath(m, "config", "auto_capture_config", "enabled").SetComputed() + - field: config.auto_capture_config.enabled + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L395-L396 + # route_optimized is ForceNew; backend returns false when not set by user. + - field: route_optimized + values: [false] registered_models: + ignore_remote_changes: + # Output-only timestamp/user fields zeroed in RemapState but still show up + # via ForceSendFields. These should never participate in diffs. + - field: created_at + reason: output_only + - field: created_by + reason: output_only + - field: updated_at + reason: output_only + - field: updated_by + reason: output_only recreate_on_changes: # The name can technically be updated without recreate. We recreate for now though # to match TF implementation. @@ -131,6 +268,14 @@ resources: reason: immutable - field: storage_location reason: immutable + backend_defaults: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/catalog/resource_registered_model.go#L28 + # m["storage_location"].Computed = true + - field: storage_location + # owner, full_name, metastore_id are Computed in TF (backend-set output fields). + - field: owner + - field: full_name + - field: metastore_id quality_monitors: recreate_on_changes: @@ -140,7 +285,11 @@ resources: reason: immutable ignore_remote_changes: - field: warehouse_id - reason: not_returned_by_api + reason: input_only + # skip_builtin_dashboard is input-only, not returned by the Get API. + # TF preserves it from state; see resource_quality_monitor.go. + - field: skip_builtin_dashboard + reason: input_only catalogs: recreate_on_changes: @@ -194,9 +343,11 @@ resources: update_id_on_changes: - field: name reason: id_changes + backend_defaults: + # storage_location is Computed; backend generates it for managed volumes. + - field: storage_location dashboards: - ignore_remote_changes: # "serialized_dashboard" locally and remotely will have different contents # We only need to rely on etag here, and can skip this field for diff computation. @@ -214,6 +365,10 @@ resources: recreate_on_changes: - field: name reason: immutable + backend_defaults: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/internal/providers/pluginfw/products/app/resource_app.go#L41 + # s["compute_size"] = s["compute_size"].SetComputed() + - field: compute_size secret_scopes: recreate_on_changes: @@ -234,6 +389,76 @@ resources: - field: scope_name reason: id_changes + clusters: + ignore_remote_changes: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L361-L363 + # s.SchemaPath("aws_attributes").SetSuppressDiff() + # s.SchemaPath("azure_attributes").SetSuppressDiff() + # s.SchemaPath("gcp_attributes").SetSuppressDiff() + - field: aws_attributes + reason: managed + - field: azure_attributes + reason: managed + - field: gcp_attributes + reason: managed + + backend_defaults: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L331 + # s.SchemaPath("enable_elastic_disk").SetComputed() + - field: enable_elastic_disk + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L332 + # s.SchemaPath("enable_local_disk_encryption").SetComputed() + - field: enable_local_disk_encryption + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L333 + # s.SchemaPath("node_type_id").SetComputed() + - field: node_type_id + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L334 + # s.SchemaPath("driver_node_type_id").SetComputed() + - field: driver_node_type_id + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L335 + # s.SchemaPath("driver_instance_pool_id").SetComputed() + - field: driver_instance_pool_id + + # Terraform currently does not do this, but it is a field with backend default. + # See https://github.com/databricks/cli/issues/4418 + - field: single_user_name + + # We have custom handler for this in cluster.go + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L109-L118 + # DataSecurityModeDiffSuppressFunc: suppress when old != "" && new == "" + #- field: data_security_mode + + sql_warehouses: + ignore_remote_changes: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/sql/resource_sql_endpoint.go#L62 + # common.CustomizeSchemaPath(m, "channel").SetSuppressDiff() + - field: channel + reason: managed + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/sql/resource_sql_endpoint.go#L82 + # common.CustomizeSchemaPath(m, "tags").SetSuppressDiff() + - field: tags + reason: managed + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/sql/resource_sql_endpoint.go#L85-L87 + # common.CustomizeSchemaPath(m, "warehouse_type").SetSuppressDiff() + - field: warehouse_type + reason: managed + + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/sql/resource_sql_endpoint.go#L75 + # common.CustomizeSchemaPath(m, "min_num_clusters").SetSuppressDiff() + - field: min_num_clusters + reason: managed + + backend_defaults: + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/sql/resource_sql_endpoint.go#L69 + # m["enable_serverless_compute"].Computed = true + - field: enable_serverless_compute + postgres_projects: recreate_on_changes: # project_id is immutable (part of hierarchical name, not in API spec) diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 6f67f8ce43..6c6e9951ba 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -245,7 +245,7 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va // For most resources state consists of fully resolved local config snapshot + id. // Dashboards are special in that they also store "etag" in state which is not provided by user but // comes from remote state. If we don't store "etag" in state, we won't detect remote drift, because - // local=nil, remote="" which will be classified as "server_side_default". + // local=nil, remote="" which will be classified as a backend default and skipped. for key := range plan.Plan { etag := etags[key] diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index 9a934324f8..ca900e97a8 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -104,11 +104,6 @@ func clusterFixUps(cluster *compute.ClusterDetails) { ) } - if cluster.DataSecurityMode == "" { - cluster.DataSecurityMode = compute.DataSecurityModeSingleUser - cluster.ForceSendFields = append(cluster.ForceSendFields, "DataSecurityMode") - } - cluster.ForceSendFields = append(cluster.ForceSendFields, "EnableElasticDisk") if cluster.DriverNodeTypeId == "" && cluster.NodeTypeId != "" {