Added support for UC external locations (direct mode only)

acceptance/bundle/refschema/out.fields.txt

@@ -662,6 +662,66 @@ resources.experiments.*.permissions.permissions[*].group_name	string	ALL
 resources.experiments.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
 resources.experiments.*.permissions.permissions[*].service_principal_name	string	ALL
 resources.experiments.*.permissions.permissions[*].user_name	string	ALL
+resources.external_locations.*.browse_only	bool	REMOTE
+resources.external_locations.*.comment	string	ALL
+resources.external_locations.*.created_at	int64	REMOTE
+resources.external_locations.*.created_by	string	REMOTE
+resources.external_locations.*.credential_id	string	REMOTE
+resources.external_locations.*.credential_name	string	ALL
+resources.external_locations.*.enable_file_events	bool	ALL
+resources.external_locations.*.encryption_details	*catalog.EncryptionDetails	ALL
+resources.external_locations.*.encryption_details.sse_encryption_details	*catalog.SseEncryptionDetails	ALL
+resources.external_locations.*.encryption_details.sse_encryption_details.algorithm	catalog.SseEncryptionDetailsAlgorithm	ALL
+resources.external_locations.*.encryption_details.sse_encryption_details.aws_kms_key_arn	string	ALL
+resources.external_locations.*.fallback	bool	ALL
+resources.external_locations.*.file_event_queue	*catalog.FileEventQueue	ALL
+resources.external_locations.*.file_event_queue.managed_aqs	*catalog.AzureQueueStorage	ALL
+resources.external_locations.*.file_event_queue.managed_aqs.managed_resource_id	string	ALL
+resources.external_locations.*.file_event_queue.managed_aqs.queue_url	string	ALL
+resources.external_locations.*.file_event_queue.managed_aqs.resource_group	string	ALL
+resources.external_locations.*.file_event_queue.managed_aqs.subscription_id	string	ALL
+resources.external_locations.*.file_event_queue.managed_pubsub	*catalog.GcpPubsub	ALL
+resources.external_locations.*.file_event_queue.managed_pubsub.managed_resource_id	string	ALL
+resources.external_locations.*.file_event_queue.managed_pubsub.subscription_name	string	ALL
+resources.external_locations.*.file_event_queue.managed_sqs	*catalog.AwsSqsQueue	ALL
+resources.external_locations.*.file_event_queue.managed_sqs.managed_resource_id	string	ALL
+resources.external_locations.*.file_event_queue.managed_sqs.queue_url	string	ALL
+resources.external_locations.*.file_event_queue.provided_aqs	*catalog.AzureQueueStorage	ALL
+resources.external_locations.*.file_event_queue.provided_aqs.managed_resource_id	string	ALL
+resources.external_locations.*.file_event_queue.provided_aqs.queue_url	string	ALL
+resources.external_locations.*.file_event_queue.provided_aqs.resource_group	string	ALL
+resources.external_locations.*.file_event_queue.provided_aqs.subscription_id	string	ALL
+resources.external_locations.*.file_event_queue.provided_pubsub	*catalog.GcpPubsub	ALL
+resources.external_locations.*.file_event_queue.provided_pubsub.managed_resource_id	string	ALL
+resources.external_locations.*.file_event_queue.provided_pubsub.subscription_name	string	ALL
+resources.external_locations.*.file_event_queue.provided_sqs	*catalog.AwsSqsQueue	ALL
+resources.external_locations.*.file_event_queue.provided_sqs.managed_resource_id	string	ALL
+resources.external_locations.*.file_event_queue.provided_sqs.queue_url	string	ALL
+resources.external_locations.*.grants	[]resources.ExternalLocationGrant	INPUT
+resources.external_locations.*.grants[*]	resources.ExternalLocationGrant	INPUT
+resources.external_locations.*.grants[*].principal	string	INPUT
+resources.external_locations.*.grants[*].privileges	[]resources.ExternalLocationGrantPrivilege	INPUT
+resources.external_locations.*.grants[*].privileges[*]	resources.ExternalLocationGrantPrivilege	INPUT
+resources.external_locations.*.id	string	INPUT
+resources.external_locations.*.isolation_mode	catalog.IsolationMode	REMOTE
+resources.external_locations.*.lifecycle	resources.Lifecycle	INPUT
+resources.external_locations.*.lifecycle.prevent_destroy	bool	INPUT
+resources.external_locations.*.metastore_id	string	REMOTE
+resources.external_locations.*.modified_status	string	INPUT
+resources.external_locations.*.name	string	ALL
+resources.external_locations.*.owner	string	REMOTE
+resources.external_locations.*.read_only	bool	ALL
+resources.external_locations.*.skip_validation	bool	INPUT	STATE
+resources.external_locations.*.updated_at	int64	REMOTE
+resources.external_locations.*.updated_by	string	REMOTE
+resources.external_locations.*.url	string	ALL
+resources.external_locations.*.grants.full_name	string	ALL
+resources.external_locations.*.grants.grants	[]dresources.GrantAssignment	ALL
+resources.external_locations.*.grants.grants[*]	dresources.GrantAssignment	ALL
+resources.external_locations.*.grants.grants[*].principal	string	ALL
+resources.external_locations.*.grants.grants[*].privileges	[]catalog.Privilege	ALL
+resources.external_locations.*.grants.grants[*].privileges[*]	catalog.Privilege	ALL
+resources.external_locations.*.grants.securable_type	string	ALL
 resources.jobs.*.budget_policy_id	string	ALL
 resources.jobs.*.continuous	*jobs.Continuous	ALL
 resources.jobs.*.continuous.pause_status	jobs.PauseStatus	ALL

acceptance/bundle/resources/catalogs_and_external_locations/databricks.yml.tmpl

@@ -0,0 +1,36 @@
+bundle:
+  name: catalog-and-ext-loc-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  catalogs:
+    test_catalog:
+      name: test_catalog_$UNIQUE_NAME
+      comment: "Test catalog for external locations"
+      properties:
+        owner: "dabs"
+      grants:
+        - principal: deco-test-user@databricks.com
+          privileges:
+            - USE_CATALOG
+            - CREATE_SCHEMA
+
+  external_locations:
+    test_location:
+      name: test_ext_location_$UNIQUE_NAME
+      url: s3://test-bucket/path
+      credential_name: test_storage_credential
+      comment: "Test external location from DABs"
+      skip_validation: true
+      read_only: false
+      grants:
+        - principal: deco-test-user@databricks.com
+          privileges:
+            - READ_FILES
+            - WRITE_FILES
+
+targets:
+  development:
+    default: true

acceptance/bundle/resources/catalogs_and_external_locations/output.txt

@@ -0,0 +1,112 @@
+
+=== Deploy bundle with catalog and external location
+>>> [CLI] bundle plan
+create catalogs.test_catalog
+create catalogs.test_catalog.grants
+create external_locations.test_location
+create external_locations.test_location.grants
+
+Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Assert the catalog is created with grants
+>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME]
+{
+  "name": "test_catalog_[UNIQUE_NAME]",
+  "comment": "Test catalog for external locations",
+  "properties": {
+    "owner": "dabs"
+  }
+}
+
+>>> [CLI] grants get catalog test_catalog_[UNIQUE_NAME]
+{
+  "privilege_assignments": [
+    {
+      "principal": "deco-test-user@databricks.com",
+      "privileges": [
+        "CREATE_SCHEMA",
+        "USE_CATALOG"
+      ]
+    }
+  ]
+}
+
+=== Assert the external location is created with grants
+>>> [CLI] external-locations get test_ext_location_[UNIQUE_NAME]
+{
+  "name": "test_ext_location_[UNIQUE_NAME]",
+  "url": "s3://test-bucket/path",
+  "credential_name": "test_storage_credential",
+  "comment": "Test external location from DABs"
+}
+
+>>> [CLI] grants get external_location test_ext_location_[UNIQUE_NAME]
+{
+  "privilege_assignments": [
+    {
+      "principal": "deco-test-user@databricks.com",
+      "privileges": [
+        "READ_FILES",
+        "WRITE_FILES"
+      ]
+    }
+  ]
+}
+
+=== Update external location comment
+=== Redeploy with updated comment
+>>> [CLI] bundle plan
+update external_locations.test_location
+
+Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Assert the external location comment is updated
+>>> [CLI] external-locations get test_ext_location_[UNIQUE_NAME]
+{
+  "name": "test_ext_location_[UNIQUE_NAME]",
+  "comment": "Updated external location from DABs"
+}
+
+=== Update catalog comment
+=== Redeploy with updated catalog comment
+>>> [CLI] bundle plan
+update catalogs.test_catalog
+update external_locations.test_location
+
+Plan: 0 to add, 2 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Assert the catalog comment is updated
+>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME]
+{
+  "name": "test_catalog_[UNIQUE_NAME]",
+  "comment": "Updated catalog for external locations"
+}
+
+=== Test cleanup
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.catalogs.test_catalog
+  delete resources.external_locations.test_location
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/catalogs_and_external_locations/script

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+CATALOG_NAME="test_catalog_${UNIQUE_NAME}"
+EXT_LOCATION_NAME="test_ext_location_${UNIQUE_NAME}"
+
+cleanup() {
+  title "Test cleanup"
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "Deploy bundle with catalog and external location"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+
+title "Assert the catalog is created with grants"
+trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}"
+trace $CLI grants get catalog "${CATALOG_NAME}" | jq --sort-keys
+
+title "Assert the external location is created with grants"
+trace $CLI external-locations get "${EXT_LOCATION_NAME}" | jq "{name, url, credential_name, comment}"
+trace $CLI grants get external_location "${EXT_LOCATION_NAME}" | jq --sort-keys
+
+title "Update external location comment"
+update_file.py databricks.yml "Test external location from DABs" "Updated external location from DABs"
+
+title "Redeploy with updated comment"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+
+title "Assert the external location comment is updated"
+trace $CLI external-locations get "${EXT_LOCATION_NAME}" | jq "{name, comment}"
+
+title "Update catalog comment"
+update_file.py databricks.yml "Test catalog for external locations" "Updated catalog for external locations"
+
+title "Redeploy with updated catalog comment"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+
+title "Assert the catalog comment is updated"
+trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment}"

acceptance/bundle/resources/catalogs_and_external_locations/test.toml

@@ -0,0 +1,14 @@
+Local = true
+# External locations require actual storage credentials with cloud IAM setup
+# which are environment-specific, so we only test locally with the mock server
+Cloud = false
+RecordRequests = false
+RequiresUnityCatalog = true
+
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+]
+
+[EnvMatrix]
+  DATABRICKS_BUNDLE_ENGINE = ["direct"]

bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go

@@ -20,6 +20,7 @@ import (
 // These resources are there because they use grants, not permissions:
 var unsupportedResources = []string{
 	"catalogs",
+	"external_locations",
 	"volumes",
 	"schemas",
 	"quality_monitors",

bundle/config/mutator/resourcemutator/apply_target_mode_test.go

@@ -135,6 +135,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 				Catalogs: map[string]*resources.Catalog{
 					"catalog1": {CreateCatalog: catalog.CreateCatalog{Name: "catalog1"}},
 				},
+				ExternalLocations: map[string]*resources.ExternalLocation{
+					"externalLocation1": {CreateExternalLocation: catalog.CreateExternalLocation{Name: "externalLocation1"}},
+				},
 				Schemas: map[string]*resources.Schema{
 					"schema1": {CreateSchema: catalog.CreateSchema{Name: "schema1"}},
 				},
@@ -405,10 +408,11 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
 	b := mockBundle(config.Development)
 
 	// UC resources should not have a prefix added to their name. Right now
-	// this list only contains the Volume and Catalog resources since we have yet to remove
+	// this list only contains the Volume, Catalog, and ExternalLocation resources since we have yet to remove
 	// prefixing support for UC schemas and registered models.
 	ucFields := []reflect.Type{
 		reflect.TypeOf(&resources.Catalog{}),
+		reflect.TypeOf(&resources.ExternalLocation{}),
 		reflect.TypeOf(&resources.Volume{}),
 	}
 

bundle/config/mutator/resourcemutator/run_as_test.go

@@ -41,6 +41,7 @@ func allResourceTypes(t *testing.T) []string {
 		"database_catalogs",
 		"database_instances",
 		"experiments",
+		"external_locations",
 		"jobs",
 		"model_serving_endpoints",
 		"models",
@@ -166,6 +167,7 @@ var allowList = []string{
 	"clusters",
 	"database_catalogs",
 	"database_instances",
+	"external_locations",
 	"synced_database_tables",
 	"jobs",
 	"pipelines",

bundle/config/mutator/validate_direct_only_resources.go

@@ -30,6 +30,18 @@ var directOnlyResources = []directOnlyResource{
 			return result
 		},
 	},
+	{
+		resourceType: "external_locations",
+		pluralName:   "External Location",
+		singularName: "external location",
+		getResources: func(b *bundle.Bundle) map[string]any {
+			result := make(map[string]any)
+			for k, v := range b.Config.Resources.ExternalLocations {
+				result[k] = v
+			}
+			return result
+		},
+	},
 }
 
 type validateDirectOnlyResources struct {

bundle/config/resources.go

@@ -22,6 +22,7 @@ type Resources struct {
 	Catalogs              map[string]*resources.Catalog              `json:"catalogs,omitempty"`
 	Schemas               map[string]*resources.Schema               `json:"schemas,omitempty"`
 	Volumes               map[string]*resources.Volume               `json:"volumes,omitempty"`
+	ExternalLocations     map[string]*resources.ExternalLocation     `json:"external_locations,omitempty"`
 	Clusters              map[string]*resources.Cluster              `json:"clusters,omitempty"`
 	Dashboards            map[string]*resources.Dashboard            `json:"dashboards,omitempty"`
 	Apps                  map[string]*resources.App                  `json:"apps,omitempty"`
@@ -93,6 +94,7 @@ func (r *Resources) AllResources() []ResourceGroup {
 		collectResourceMap(descriptions["quality_monitors"], r.QualityMonitors),
 		collectResourceMap(descriptions["catalogs"], r.Catalogs),
 		collectResourceMap(descriptions["schemas"], r.Schemas),
+		collectResourceMap(descriptions["external_locations"], r.ExternalLocations),
 		collectResourceMap(descriptions["clusters"], r.Clusters),
 		collectResourceMap(descriptions["dashboards"], r.Dashboards),
 		collectResourceMap(descriptions["volumes"], r.Volumes),
@@ -141,6 +143,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
 		}
 	}
 
+	for k := range r.ExternalLocations {
+		if k == key {
+			found = append(found, r.ExternalLocations[k])
+		}
+	}
+
 	for k := range r.Experiments {
 		if k == key {
 			found = append(found, r.Experiments[k])
@@ -264,6 +272,7 @@ func SupportedResources() map[string]resources.ResourceDescription {
 		"quality_monitors":        (&resources.QualityMonitor{}).ResourceDescription(),
 		"catalogs":                (&resources.Catalog{}).ResourceDescription(),
 		"schemas":                 (&resources.Schema{}).ResourceDescription(),
+		"external_locations":      (&resources.ExternalLocation{}).ResourceDescription(),
 		"clusters":                (&resources.Cluster{}).ResourceDescription(),
 		"dashboards":              (&resources.Dashboard{}).ResourceDescription(),
 		"volumes":                 (&resources.Volume{}).ResourceDescription(),