From 763c38b9548c65b581cbaa9eb01c507b4d2f5bd1 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Feb 2026 11:25:50 +0000 Subject: [PATCH 1/8] Added support for UC external locations (direct mode only) --- .../databricks.yml.tmpl | 36 +++++ .../out.test.toml | 6 + .../output.txt | 112 ++++++++++++++ .../catalogs_and_external_locations/script | 44 ++++++ .../catalogs_and_external_locations/test.toml | 14 ++ .../mutator/validate_direct_only_resources.go | 12 ++ bundle/config/resources.go | 9 ++ bundle/config/resources/external_location.go | 108 ++++++++++++++ .../resources/external_location_test.go | 65 +++++++++ bundle/direct/dresources/all.go | 10 +- bundle/direct/dresources/external_location.go | 124 ++++++++++++++++ bundle/direct/dresources/grants.go | 9 +- bundle/direct/dresources/resources.yml | 12 ++ bundle/statemgmt/state_load_test.go | 23 +++ libs/structs/structwalk/walktype_test.go | 2 +- libs/testserver/external_locations.go | 137 ++++++++++++++++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 18 +++ 18 files changed, 734 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/resources/catalogs_and_external_locations/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs_and_external_locations/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs_and_external_locations/output.txt create mode 100755 acceptance/bundle/resources/catalogs_and_external_locations/script create mode 100644 acceptance/bundle/resources/catalogs_and_external_locations/test.toml create mode 100644 bundle/config/resources/external_location.go create mode 100644 bundle/config/resources/external_location_test.go create mode 100644 bundle/direct/dresources/external_location.go create mode 100644 libs/testserver/external_locations.go diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs_and_external_locations/databricks.yml.tmpl new file mode 100644 index 0000000000..5e20090229 --- /dev/null +++ b/acceptance/bundle/resources/catalogs_and_external_locations/databricks.yml.tmpl @@ -0,0 +1,36 @@ +bundle: + name: catalog-and-ext-loc-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + catalogs: + test_catalog: + name: test_catalog_$UNIQUE_NAME + comment: "Test catalog for external locations" + properties: + owner: "dabs" + grants: + - principal: deco-test-user@databricks.com + privileges: + - USE_CATALOG + - CREATE_SCHEMA + + external_locations: + test_location: + name: test_ext_location_$UNIQUE_NAME + url: s3://test-bucket/path + credential_name: test_storage_credential + comment: "Test external location from DABs" + skip_validation: true + read_only: false + grants: + - principal: deco-test-user@databricks.com + privileges: + - READ_FILES + - WRITE_FILES + +targets: + development: + default: true diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/out.test.toml b/acceptance/bundle/resources/catalogs_and_external_locations/out.test.toml new file mode 100644 index 0000000000..5566892a0d --- /dev/null +++ b/acceptance/bundle/resources/catalogs_and_external_locations/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/output.txt b/acceptance/bundle/resources/catalogs_and_external_locations/output.txt new file mode 100644 index 0000000000..38ee94ca10 --- /dev/null +++ b/acceptance/bundle/resources/catalogs_and_external_locations/output.txt @@ -0,0 +1,112 @@ + +=== Deploy bundle with catalog and external location +>>> [CLI] bundle plan +create catalogs.test_catalog +create catalogs.test_catalog.grants +create external_locations.test_location +create external_locations.test_location.grants + +Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert the catalog is created with grants +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "comment": "Test catalog for external locations", + "properties": { + "owner": "dabs" + } +} + +>>> [CLI] grants get catalog test_catalog_[UNIQUE_NAME] +{ + "privilege_assignments": [ + { + "principal": "deco-test-user@databricks.com", + "privileges": [ + "CREATE_SCHEMA", + "USE_CATALOG" + ] + } + ] +} + +=== Assert the external location is created with grants +>>> [CLI] external-locations get test_ext_location_[UNIQUE_NAME] +{ + "name": "test_ext_location_[UNIQUE_NAME]", + "url": "s3://test-bucket/path", + "credential_name": "test_storage_credential", + "comment": "Test external location from DABs" +} + +>>> [CLI] grants get external_location test_ext_location_[UNIQUE_NAME] +{ + "privilege_assignments": [ + { + "principal": "deco-test-user@databricks.com", + "privileges": [ + "READ_FILES", + "WRITE_FILES" + ] + } + ] +} + +=== Update external location comment +=== Redeploy with updated comment +>>> [CLI] bundle plan +update external_locations.test_location + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert the external location comment is updated +>>> [CLI] external-locations get test_ext_location_[UNIQUE_NAME] +{ + "name": "test_ext_location_[UNIQUE_NAME]", + "comment": "Updated external location from DABs" +} + +=== Update catalog comment +=== Redeploy with updated catalog comment +>>> [CLI] bundle plan +update catalogs.test_catalog +update external_locations.test_location + +Plan: 0 to add, 2 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert the catalog comment is updated +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "comment": "Updated catalog for external locations" +} + +=== Test cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.test_catalog + delete resources.external_locations.test_location + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/script b/acceptance/bundle/resources/catalogs_and_external_locations/script new file mode 100755 index 0000000000..1ff7d02ec4 --- /dev/null +++ b/acceptance/bundle/resources/catalogs_and_external_locations/script @@ -0,0 +1,44 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +CATALOG_NAME="test_catalog_${UNIQUE_NAME}" +EXT_LOCATION_NAME="test_ext_location_${UNIQUE_NAME}" + +cleanup() { + title "Test cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy bundle with catalog and external location" +trace $CLI bundle plan +trace $CLI bundle deploy + +title "Assert the catalog is created with grants" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}" +trace $CLI grants get catalog "${CATALOG_NAME}" | jq --sort-keys + +title "Assert the external location is created with grants" +trace $CLI external-locations get "${EXT_LOCATION_NAME}" | jq "{name, url, credential_name, comment}" +trace $CLI grants get external_location "${EXT_LOCATION_NAME}" | jq --sort-keys + +title "Update external location comment" +update_file.py databricks.yml "Test external location from DABs" "Updated external location from DABs" + +title "Redeploy with updated comment" +trace $CLI bundle plan +trace $CLI bundle deploy + +title "Assert the external location comment is updated" +trace $CLI external-locations get "${EXT_LOCATION_NAME}" | jq "{name, comment}" + +title "Update catalog comment" +update_file.py databricks.yml "Test catalog for external locations" "Updated catalog for external locations" + +title "Redeploy with updated catalog comment" +trace $CLI bundle plan +trace $CLI bundle deploy + +title "Assert the catalog comment is updated" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment}" diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/test.toml b/acceptance/bundle/resources/catalogs_and_external_locations/test.toml new file mode 100644 index 0000000000..55cc7b681f --- /dev/null +++ b/acceptance/bundle/resources/catalogs_and_external_locations/test.toml @@ -0,0 +1,14 @@ +Local = true +# External locations require actual storage credentials with cloud IAM setup +# which are environment-specific, so we only test locally with the mock server +Cloud = false +RecordRequests = false +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_direct_only_resources.go b/bundle/config/mutator/validate_direct_only_resources.go index e39153d987..1ee0f66d08 100644 --- a/bundle/config/mutator/validate_direct_only_resources.go +++ b/bundle/config/mutator/validate_direct_only_resources.go @@ -30,6 +30,18 @@ var directOnlyResources = []directOnlyResource{ return result }, }, + { + resourceType: "external_locations", + pluralName: "External Location", + singularName: "external location", + getResources: func(b *bundle.Bundle) map[string]any { + result := make(map[string]any) + for k, v := range b.Config.Resources.ExternalLocations { + result[k] = v + } + return result + }, + }, } type validateDirectOnlyResources struct { diff --git a/bundle/config/resources.go b/bundle/config/resources.go index b1c6a37bba..272b4515bc 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -22,6 +22,7 @@ type Resources struct { Catalogs map[string]*resources.Catalog `json:"catalogs,omitempty"` Schemas map[string]*resources.Schema `json:"schemas,omitempty"` Volumes map[string]*resources.Volume `json:"volumes,omitempty"` + ExternalLocations map[string]*resources.ExternalLocation `json:"external_locations,omitempty"` Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` Apps map[string]*resources.App `json:"apps,omitempty"` @@ -93,6 +94,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["quality_monitors"], r.QualityMonitors), collectResourceMap(descriptions["catalogs"], r.Catalogs), collectResourceMap(descriptions["schemas"], r.Schemas), + collectResourceMap(descriptions["external_locations"], r.ExternalLocations), collectResourceMap(descriptions["clusters"], r.Clusters), collectResourceMap(descriptions["dashboards"], r.Dashboards), collectResourceMap(descriptions["volumes"], r.Volumes), @@ -141,6 +143,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) } } + for k := range r.ExternalLocations { + if k == key { + found = append(found, r.ExternalLocations[k]) + } + } + for k := range r.Experiments { if k == key { found = append(found, r.Experiments[k]) @@ -264,6 +272,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), "catalogs": (&resources.Catalog{}).ResourceDescription(), "schemas": (&resources.Schema{}).ResourceDescription(), + "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), "clusters": (&resources.Cluster{}).ResourceDescription(), "dashboards": (&resources.Dashboard{}).ResourceDescription(), "volumes": (&resources.Volume{}).ResourceDescription(), diff --git a/bundle/config/resources/external_location.go b/bundle/config/resources/external_location.go new file mode 100644 index 0000000000..f7a79f0d05 --- /dev/null +++ b/bundle/config/resources/external_location.go @@ -0,0 +1,108 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" + + "github.com/databricks/cli/libs/log" +) + +type ExternalLocationGrantPrivilege string + +const ( + ExternalLocationGrantPrivilegeAllPrivileges ExternalLocationGrantPrivilege = "ALL_PRIVILEGES" + ExternalLocationGrantPrivilegeCreateExternalTable ExternalLocationGrantPrivilege = "CREATE_EXTERNAL_TABLE" + ExternalLocationGrantPrivilegeCreateExternalVolume ExternalLocationGrantPrivilege = "CREATE_EXTERNAL_VOLUME" + ExternalLocationGrantPrivilegeCreateManagedStorage ExternalLocationGrantPrivilege = "CREATE_MANAGED_STORAGE" + ExternalLocationGrantPrivilegeCreateTable ExternalLocationGrantPrivilege = "CREATE_TABLE" + ExternalLocationGrantPrivilegeCreateVolume ExternalLocationGrantPrivilege = "CREATE_VOLUME" + ExternalLocationGrantPrivilegeManage ExternalLocationGrantPrivilege = "MANAGE" + ExternalLocationGrantPrivilegeReadFiles ExternalLocationGrantPrivilege = "READ_FILES" + ExternalLocationGrantPrivilegeWriteFiles ExternalLocationGrantPrivilege = "WRITE_FILES" +) + +// Values returns all valid ExternalLocationGrantPrivilege values +func (ExternalLocationGrantPrivilege) Values() []ExternalLocationGrantPrivilege { + return []ExternalLocationGrantPrivilege{ + ExternalLocationGrantPrivilegeAllPrivileges, + ExternalLocationGrantPrivilegeCreateExternalTable, + ExternalLocationGrantPrivilegeCreateExternalVolume, + ExternalLocationGrantPrivilegeCreateManagedStorage, + ExternalLocationGrantPrivilegeCreateTable, + ExternalLocationGrantPrivilegeCreateVolume, + ExternalLocationGrantPrivilegeManage, + ExternalLocationGrantPrivilegeReadFiles, + ExternalLocationGrantPrivilegeWriteFiles, + } +} + +// ExternalLocationGrant holds the grant level settings for a single principal in Unity Catalog. +// Multiple of these can be defined on any external location. +type ExternalLocationGrant struct { + Privileges []ExternalLocationGrantPrivilege `json:"privileges"` + + Principal string `json:"principal"` +} + +type ExternalLocation struct { + // Manually include BaseResource fields to avoid URL field conflict + ID string `json:"id,omitempty" bundle:"readonly"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + // Note: We intentionally don't include BaseResource.URL here to avoid conflict with Url field below + Lifecycle Lifecycle `json:"lifecycle,omitempty"` + + catalog.CreateExternalLocation + + // List of grants to apply on this external location. + Grants []ExternalLocationGrant `json:"grants,omitempty"` +} + +func (e *ExternalLocation) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.ExternalLocations.GetByName(ctx, name) + if err != nil { + log.Debugf(ctx, "external location with name %s does not exist: %v", name, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + return true, nil +} + +func (*ExternalLocation) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "external location", + PluralName: "external_locations", + SingularTitle: "External Location", + PluralTitle: "External Locations", + } +} + +func (e *ExternalLocation) InitializeURL(baseURL url.URL) { + // External locations don't have a workspace URL + // The Url field is for the storage path (s3://...), not a workspace URL +} + +func (e *ExternalLocation) GetURL() string { + // Return empty as external locations don't have a workspace URL + return "" +} + +func (e *ExternalLocation) GetName() string { + return e.Name +} + +func (e *ExternalLocation) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, e) +} + +func (e ExternalLocation) MarshalJSON() ([]byte, error) { + return marshal.Marshal(e) +} diff --git a/bundle/config/resources/external_location_test.go b/bundle/config/resources/external_location_test.go new file mode 100644 index 0000000000..69f1ca2af2 --- /dev/null +++ b/bundle/config/resources/external_location_test.go @@ -0,0 +1,65 @@ +package resources + +import ( + "context" + "testing" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestExternalLocationExists(t *testing.T) { + ctx := context.Background() + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockExternalLocationsAPI() + + t.Run("exists", func(t *testing.T) { + api.EXPECT(). + GetByName(mock.Anything, "test_location"). + Return(&catalog.ExternalLocationInfo{Name: "test_location"}, nil) + + el := &ExternalLocation{} + exists, err := el.Exists(ctx, m.WorkspaceClient, "test_location") + require.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("does not exist", func(t *testing.T) { + notFoundErr := &apierr.APIError{ + StatusCode: 404, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + } + api.EXPECT(). + GetByName(mock.Anything, "nonexistent"). + Return(nil, notFoundErr) + + el := &ExternalLocation{} + exists, err := el.Exists(ctx, m.WorkspaceClient, "nonexistent") + require.NoError(t, err) + assert.False(t, exists) + }) +} + +func TestExternalLocationResourceDescription(t *testing.T) { + el := &ExternalLocation{} + desc := el.ResourceDescription() + + assert.Equal(t, "external location", desc.SingularName) + assert.Equal(t, "external_locations", desc.PluralName) + assert.Equal(t, "External Location", desc.SingularTitle) + assert.Equal(t, "External Locations", desc.PluralTitle) +} + +func TestExternalLocationGetName(t *testing.T) { + el := &ExternalLocation{ + CreateExternalLocation: catalog.CreateExternalLocation{ + Name: "my_location", + }, + } + + assert.Equal(t, "my_location", el.GetName()) +} diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 503905d199..56da049ab2 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -12,6 +12,7 @@ var SupportedResources = map[string]any{ "experiments": (*ResourceExperiment)(nil), "catalogs": (*ResourceCatalog)(nil), "schemas": (*ResourceSchema)(nil), + "external_locations": (*ResourceExternalLocation)(nil), "volumes": (*ResourceVolume)(nil), "models": (*ResourceMlflowModel)(nil), "apps": (*ResourceApp)(nil), @@ -45,10 +46,11 @@ var SupportedResources = map[string]any{ "dashboards.permissions": (*ResourcePermissions)(nil), // Grants - "catalogs.grants": (*ResourceGrants)(nil), - "schemas.grants": (*ResourceGrants)(nil), - "volumes.grants": (*ResourceGrants)(nil), - "registered_models.grants": (*ResourceGrants)(nil), + "catalogs.grants": (*ResourceGrants)(nil), + "schemas.grants": (*ResourceGrants)(nil), + "external_locations.grants": (*ResourceGrants)(nil), + "volumes.grants": (*ResourceGrants)(nil), + "registered_models.grants": (*ResourceGrants)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/external_location.go b/bundle/direct/dresources/external_location.go new file mode 100644 index 0000000000..e7d9def439 --- /dev/null +++ b/bundle/direct/dresources/external_location.go @@ -0,0 +1,124 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type ResourceExternalLocation struct { + client *databricks.WorkspaceClient +} + +func (*ResourceExternalLocation) New(client *databricks.WorkspaceClient) *ResourceExternalLocation { + return &ResourceExternalLocation{client: client} +} + +func (*ResourceExternalLocation) PrepareState(input *resources.ExternalLocation) *catalog.CreateExternalLocation { + return &input.CreateExternalLocation +} + +func (*ResourceExternalLocation) RemapState(info *catalog.ExternalLocationInfo) *catalog.CreateExternalLocation { + return &catalog.CreateExternalLocation{ + Comment: info.Comment, + CredentialName: info.CredentialName, + EnableFileEvents: info.EnableFileEvents, + EncryptionDetails: info.EncryptionDetails, + Fallback: info.Fallback, + FileEventQueue: info.FileEventQueue, + Name: info.Name, + ReadOnly: info.ReadOnly, + // Note: SkipValidation is input-only and not included in state + Url: info.Url, + ForceSendFields: utils.FilterFields[catalog.CreateExternalLocation](info.ForceSendFields), + } +} + +func (r *ResourceExternalLocation) DoRead(ctx context.Context, id string) (*catalog.ExternalLocationInfo, error) { + return r.client.ExternalLocations.GetByName(ctx, id) +} + +func (r *ResourceExternalLocation) DoCreate(ctx context.Context, config *catalog.CreateExternalLocation) (string, *catalog.ExternalLocationInfo, error) { + response, err := r.client.ExternalLocations.Create(ctx, *config) + if err != nil || response == nil { + return "", nil, err + } + return response.Name, response, nil +} + +// DoUpdate updates the external location in place and returns remote state. +func (r *ResourceExternalLocation) DoUpdate(ctx context.Context, id string, config *catalog.CreateExternalLocation, _ Changes) (*catalog.ExternalLocationInfo, error) { + updateRequest := catalog.UpdateExternalLocation{ + Comment: config.Comment, + CredentialName: config.CredentialName, + EnableFileEvents: config.EnableFileEvents, + // EncryptionDetails is not supported for updates + Fallback: config.Fallback, + // FileEventQueue is not supported for updates + Force: false, + IsolationMode: "", // Not supported by DABs + Name: id, + NewName: "", // Only set if name actually changes (see DoUpdateWithID) + Owner: "", // Not supported by DABs + ReadOnly: config.ReadOnly, + SkipValidation: config.SkipValidation, + Url: config.Url, + ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](config.ForceSendFields, "IsolationMode", "Owner"), + } + + response, err := r.client.ExternalLocations.Update(ctx, updateRequest) + if err != nil { + return nil, err + } + + return response, nil +} + +// DoUpdateWithID updates the external location and returns the new ID if the name changes. +func (r *ResourceExternalLocation) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateExternalLocation) (string, *catalog.ExternalLocationInfo, error) { + updateRequest := catalog.UpdateExternalLocation{ + Comment: config.Comment, + CredentialName: config.CredentialName, + EnableFileEvents: config.EnableFileEvents, + // EncryptionDetails is not supported for updates + Fallback: config.Fallback, + // FileEventQueue is not supported for updates + Force: false, + IsolationMode: "", // Not supported by DABs + Name: id, + NewName: "", // Initialized below if needed + Owner: "", // Not supported by DABs + ReadOnly: config.ReadOnly, + SkipValidation: config.SkipValidation, + Url: config.Url, + ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](config.ForceSendFields, "IsolationMode", "Owner"), + } + + if config.Name != id { + updateRequest.NewName = config.Name + } + + response, err := r.client.ExternalLocations.Update(ctx, updateRequest) + if err != nil { + return "", nil, err + } + + // Return the new name as the ID if it changed, otherwise return the old ID + newID := id + if updateRequest.NewName != "" { + newID = updateRequest.NewName + } + + return newID, response, nil +} + +func (r *ResourceExternalLocation) DoDelete(ctx context.Context, id string) error { + return r.client.ExternalLocations.Delete(ctx, catalog.DeleteExternalLocationRequest{ + Name: id, + Force: true, + ForceSendFields: nil, + }) +} diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index dcdc882e0c..76e504e5a0 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -14,10 +14,11 @@ import ( ) var grantResourceToSecurableType = map[string]string{ - "catalogs": "catalog", - "schemas": "schema", - "volumes": "volume", - "registered_models": "function", + "catalogs": "catalog", + "schemas": "schema", + "external_locations": "external_location", + "volumes": "volume", + "registered_models": "function", } type GrantAssignment struct { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 740628978a..c49f07de63 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -130,6 +130,18 @@ resources: - field: storage_root reason: immutable + external_locations: + recreate_on_changes: + - field: credential_name + reason: immutable + - field: encryption_details + reason: immutable + - field: file_event_queue + reason: immutable + update_id_on_changes: + - field: name + reason: id_changes + volumes: recreate_on_changes: - field: catalog_name diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index b60b7472a6..466e966550 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -35,6 +35,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.quality_monitors.test_monitor": {ID: "1"}, "resources.catalogs.test_catalog": {ID: "1"}, "resources.schemas.test_schema": {ID: "1"}, + "resources.external_locations.test_external_location": {ID: "1"}, "resources.volumes.test_volume": {ID: "1"}, "resources.clusters.test_cluster": {ID: "1"}, "resources.dashboards.test_dashboard": {ID: "1"}, @@ -79,6 +80,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.ExternalLocations["test_external_location"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ExternalLocations["test_external_location"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Volumes["test_volume"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Volumes["test_volume"].ModifiedStatus) @@ -182,6 +186,14 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + ExternalLocations: map[string]*resources.ExternalLocation{ + "test_external_location": { + CreateExternalLocation: catalog.CreateExternalLocation{ + Name: "test_external_location", + Url: "s3://test-bucket/path", + }, + }, + }, Volumes: map[string]*resources.Volume{ "test_volume": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ @@ -309,6 +321,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.Schemas["test_schema"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "", config.Resources.ExternalLocations["test_external_location"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ExternalLocations["test_external_location"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Volumes["test_volume"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Volumes["test_volume"].ModifiedStatus) @@ -462,6 +477,14 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + ExternalLocations: map[string]*resources.ExternalLocation{ + "test_external_location": { + CreateExternalLocation: catalog.CreateExternalLocation{ + Name: "test_external_location", + Url: "s3://test-bucket/path", + }, + }, + }, Volumes: map[string]*resources.Volume{ "test_volume": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{ diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index 0ad985cf71..38759aa151 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -136,7 +136,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeOf(config.Root{}), - 4300, 4700, // 4322 at the time of the update + 4300, 4750, // 4739 after adding external locations support map[string]any{ "bundle.target": "", `variables.*.lookup.dashboard`: "", diff --git a/libs/testserver/external_locations.go b/libs/testserver/external_locations.go new file mode 100644 index 0000000000..aed92c6644 --- /dev/null +++ b/libs/testserver/external_locations.go @@ -0,0 +1,137 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +func (s *FakeWorkspace) ExternalLocationsCreate(req Request) Response { + defer s.LockUnlock()() + + var createRequest catalog.CreateExternalLocation + if err := json.Unmarshal(req.Body, &createRequest); err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + if createRequest.Url == "" { + return Response{ + Body: "CreateExternalLocation Missing required field: url", + StatusCode: http.StatusBadRequest, + } + } + + locationInfo := catalog.ExternalLocationInfo{ + Name: createRequest.Name, + Url: createRequest.Url, + CredentialName: createRequest.CredentialName, + Comment: createRequest.Comment, + ReadOnly: createRequest.ReadOnly, + EnableFileEvents: createRequest.EnableFileEvents, + Fallback: createRequest.Fallback, + EncryptionDetails: createRequest.EncryptionDetails, + FileEventQueue: createRequest.FileEventQueue, + CreatedAt: time.Now().UnixMilli(), + CreatedBy: s.CurrentUser().UserName, + UpdatedAt: time.Now().UnixMilli(), + UpdatedBy: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + Owner: s.CurrentUser().UserName, + } + + s.ExternalLocations[createRequest.Name] = locationInfo + return Response{ + Body: locationInfo, + } +} + +func (s *FakeWorkspace) ExternalLocationsUpdate(req Request, name string) Response { + defer s.LockUnlock()() + + existing, ok := s.ExternalLocations[name] + if !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: fmt.Sprintf("external location %s not found", name), + } + } + + var updateRequest catalog.UpdateExternalLocation + if err := json.Unmarshal(req.Body, &updateRequest); err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + // Update only the fields that can be updated + if updateRequest.Comment != "" { + existing.Comment = updateRequest.Comment + } + if updateRequest.Url != "" { + existing.Url = updateRequest.Url + } + if updateRequest.CredentialName != "" { + existing.CredentialName = updateRequest.CredentialName + } + if updateRequest.Owner != "" { + existing.Owner = updateRequest.Owner + } + existing.ReadOnly = updateRequest.ReadOnly + existing.EnableFileEvents = updateRequest.EnableFileEvents + existing.Fallback = updateRequest.Fallback + + if updateRequest.NewName != "" { + existing.Name = updateRequest.NewName + + // Delete the old entry and create with new name + delete(s.ExternalLocations, name) + name = updateRequest.NewName + } + + existing.UpdatedAt = time.Now().UnixMilli() + existing.UpdatedBy = s.CurrentUser().UserName + + s.ExternalLocations[name] = existing + return Response{ + Body: existing, + } +} + +func (s *FakeWorkspace) ExternalLocationsGet(_ Request, name string) Response { + defer s.LockUnlock()() + + existing, ok := s.ExternalLocations[name] + if !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: fmt.Sprintf("external location %s not found", name), + } + } + + return Response{ + Body: existing, + } +} + +func (s *FakeWorkspace) ExternalLocationsDelete(_ Request, name string) Response { + defer s.LockUnlock()() + + if _, ok := s.ExternalLocations[name]; !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: fmt.Sprintf("external location %s not found", name), + } + } + + delete(s.ExternalLocations, name) + return Response{ + StatusCode: http.StatusOK, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 91b25ea967..aaa6e66b1a 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -143,6 +143,7 @@ type FakeWorkspace struct { ModelRegistryModels map[string]ml.Model Clusters map[string]compute.ClusterDetails Catalogs map[string]catalog.CatalogInfo + ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo ServingEndpoints map[string]serving.ServingEndpointDetailed @@ -260,6 +261,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, Catalogs: map[string]catalog.CatalogInfo{}, + ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{}, Volumes: map[string]catalog.VolumeInfo{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 9bcdce4f1f..15c720f508 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -433,6 +433,24 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.Catalogs, req.Vars["name"]) }) + // External Locations: + + server.Handle("GET", "/api/2.1/unity-catalog/external-locations/{name}", func(req Request) any { + return req.Workspace.ExternalLocationsGet(req, req.Vars["name"]) + }) + + server.Handle("POST", "/api/2.1/unity-catalog/external-locations", func(req Request) any { + return req.Workspace.ExternalLocationsCreate(req) + }) + + server.Handle("PATCH", "/api/2.1/unity-catalog/external-locations/{name}", func(req Request) any { + return req.Workspace.ExternalLocationsUpdate(req, req.Vars["name"]) + }) + + server.Handle("DELETE", "/api/2.1/unity-catalog/external-locations/{name}", func(req Request) any { + return req.Workspace.ExternalLocationsDelete(req, req.Vars["name"]) + }) + // Registered Models: server.Handle("GET", "/api/2.1/unity-catalog/models/{full_name}", func(req Request) any { From a6bdbfcce4fece75e6fc89150b508533551aaa8e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Feb 2026 12:31:26 +0000 Subject: [PATCH 2/8] fixes --- .../mutator/resourcemutator/run_as_test.go | 1 + bundle/deploy/terraform/lifecycle_test.go | 1 + bundle/internal/schema/annotations.yml | 47 ++++ .../schema/annotations_openapi_overrides.yml | 57 ++++ .../validation/generated/enum_fields.go | 3 + .../validation/generated/required_fields.go | 3 + bundle/schema/jsonschema.json | 265 ++++++++++++++++++ 7 files changed, 377 insertions(+) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index f35955a450..3c62af9341 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -166,6 +166,7 @@ var allowList = []string{ "clusters", "database_catalogs", "database_instances", + "external_locations", "synced_database_tables", "jobs", "pipelines", diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 028b328645..802a7f1bd8 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { // Resources that are only supported in direct mode and should not be converted to Terraform ignoredResources := []string{ "catalogs", + "external_locations", } for resourceType := range supportedResources { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 6464e26cac..41e1769d2c 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -186,6 +186,9 @@ github.com/databricks/cli/bundle/config.Resources: The experiment definitions for the bundle, where each key is the name of the experiment. "markdown_description": |- The experiment definitions for the bundle, where each key is the name of the experiment. See [\_](/dev-tools/bundles/resources.md#experiments). + "external_locations": + "description": |- + PLACEHOLDER "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. @@ -690,6 +693,50 @@ github.com/databricks/cli/bundle/config/resources.DatabaseInstancePermission: "user_name": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.ExternalLocation: + "comment": + "description": |- + PLACEHOLDER + "credential_name": + "description": |- + PLACEHOLDER + "enable_file_events": + "description": |- + PLACEHOLDER + "encryption_details": + "description": |- + PLACEHOLDER + "fallback": + "description": |- + PLACEHOLDER + "file_event_queue": + "description": |- + PLACEHOLDER + "grants": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "name": + "description": |- + PLACEHOLDER + "read_only": + "description": |- + PLACEHOLDER + "skip_validation": + "description": |- + PLACEHOLDER + "url": + "description": |- + PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.ExternalLocationGrant: + "principal": + "description": |- + PLACEHOLDER + "privileges": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Grant: "principal": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 737e698f66..f53ae72ebc 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -1094,3 +1094,60 @@ github.com/databricks/databricks-sdk-go/service/sql.EndpointTags: "custom_tags": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/catalog.AwsSqsQueue: + "managed_resource_id": + "description": |- + PLACEHOLDER + "queue_url": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/catalog.AzureQueueStorage: + "managed_resource_id": + "description": |- + PLACEHOLDER + "queue_url": + "description": |- + PLACEHOLDER + "resource_group": + "description": |- + PLACEHOLDER + "subscription_id": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/catalog.EncryptionDetails: + "sse_encryption_details": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/catalog.FileEventQueue: + "managed_aqs": + "description": |- + PLACEHOLDER + "managed_pubsub": + "description": |- + PLACEHOLDER + "managed_sqs": + "description": |- + PLACEHOLDER + "provided_aqs": + "description": |- + PLACEHOLDER + "provided_pubsub": + "description": |- + PLACEHOLDER + "provided_sqs": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/catalog.GcpPubsub: + "managed_resource_id": + "description": |- + PLACEHOLDER + "subscription_name": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/catalog.SseEncryptionDetails: + "algorithm": + "description": |- + PLACEHOLDER + "aws_kms_key_arn": + "description": |- + PLACEHOLDER diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index cfab6370a8..0bad78bd98 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -47,6 +47,9 @@ var EnumFields = map[string][]string{ "resources.database_instances.*.state": {"AVAILABLE", "DELETING", "FAILING_OVER", "STARTING", "STOPPED", "UPDATING"}, + "resources.external_locations.*.encryption_details.sse_encryption_details.algorithm": {"AWS_SSE_KMS", "AWS_SSE_S3"}, + "resources.external_locations.*.grants[*].privileges[*]": {"ALL_PRIVILEGES", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_MANAGED_STORAGE", "CREATE_TABLE", "CREATE_VOLUME", "MANAGE", "READ_FILES", "WRITE_FILES"}, + "resources.jobs.*.continuous.pause_status": {"PAUSED", "UNPAUSED"}, "resources.jobs.*.continuous.task_retry_mode": {"NEVER", "ON_FAILURE"}, "resources.jobs.*.deployment.kind": {"BUNDLE", "SYSTEM_MANAGED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 21bc5655a7..6d9d059468 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -60,6 +60,9 @@ var RequiredFields = map[string][]string{ "resources.experiments.*": {"name"}, "resources.experiments.*.permissions[*]": {"level"}, + "resources.external_locations.*": {"credential_name", "name", "url"}, + "resources.external_locations.*.grants[*]": {"privileges", "principal"}, + "resources.jobs.*.deployment": {"kind"}, "resources.jobs.*.environments[*]": {"environment_key"}, "resources.jobs.*.git_source": {"git_provider", "git_url"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 0a25ac5a95..f898174a56 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -896,6 +896,88 @@ } ] }, + "resources.ExternalLocation": { + "oneOf": [ + { + "type": "object", + "properties": { + "comment": { + "$ref": "#/$defs/string" + }, + "credential_name": { + "$ref": "#/$defs/string" + }, + "enable_file_events": { + "$ref": "#/$defs/bool" + }, + "encryption_details": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.EncryptionDetails" + }, + "fallback": { + "$ref": "#/$defs/bool" + }, + "file_event_queue": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.FileEventQueue" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.ExternalLocationGrant" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "name": { + "$ref": "#/$defs/string" + }, + "read_only": { + "$ref": "#/$defs/bool" + }, + "skip_validation": { + "$ref": "#/$defs/bool" + }, + "url": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "credential_name", + "name", + "url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ExternalLocationGrant": { + "oneOf": [ + { + "type": "object", + "properties": { + "principal": { + "$ref": "#/$defs/string" + }, + "privileges": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.ExternalLocationGrantPrivilege" + } + }, + "additionalProperties": false, + "required": [ + "privileges", + "principal" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ExternalLocationGrantPrivilege": { + "type": "string" + }, "resources.Grant": { "oneOf": [ { @@ -2780,6 +2862,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment", "markdownDescription": "The experiment definitions for the bundle, where each key is the name of the experiment. See [experiments](https://docs.databricks.com/dev-tools/bundles/resources.html#experiments)." }, + "external_locations": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -3748,6 +3833,121 @@ } ] }, + "catalog.AwsSqsQueue": { + "oneOf": [ + { + "type": "object", + "properties": { + "managed_resource_id": { + "$ref": "#/$defs/string" + }, + "queue_url": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.AzureQueueStorage": { + "oneOf": [ + { + "type": "object", + "properties": { + "managed_resource_id": { + "$ref": "#/$defs/string" + }, + "queue_url": { + "$ref": "#/$defs/string" + }, + "resource_group": { + "$ref": "#/$defs/string" + }, + "subscription_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.EncryptionDetails": { + "oneOf": [ + { + "type": "object", + "properties": { + "sse_encryption_details": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.SseEncryptionDetails" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.FileEventQueue": { + "oneOf": [ + { + "type": "object", + "properties": { + "managed_aqs": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.AzureQueueStorage" + }, + "managed_pubsub": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.GcpPubsub" + }, + "managed_sqs": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.AwsSqsQueue" + }, + "provided_aqs": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.AzureQueueStorage" + }, + "provided_pubsub": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.GcpPubsub" + }, + "provided_sqs": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.AwsSqsQueue" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.GcpPubsub": { + "oneOf": [ + { + "type": "object", + "properties": { + "managed_resource_id": { + "$ref": "#/$defs/string" + }, + "subscription_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "catalog.MonitorCronSchedule": { "oneOf": [ { @@ -4053,6 +4253,29 @@ } ] }, + "catalog.SseEncryptionDetails": { + "oneOf": [ + { + "type": "object", + "properties": { + "algorithm": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.SseEncryptionDetailsAlgorithm" + }, + "aws_kms_key_arn": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.SseEncryptionDetailsAlgorithm": { + "type": "string" + }, "catalog.VolumeType": { "oneOf": [ { @@ -10370,6 +10593,20 @@ } ] }, + "resources.ExternalLocation": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocation" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -10802,6 +11039,34 @@ } ] }, + "resources.ExternalLocationGrant": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocationGrant" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ExternalLocationGrantPrivilege": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocationGrantPrivilege" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Grant": { "oneOf": [ { From ed49e6b748d67cf3a4781773ae9cf1b6b6c7aeab Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Feb 2026 13:10:57 +0000 Subject: [PATCH 3/8] fixed tests --- .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 6 ++- .../mutator/resourcemutator/run_as_test.go | 1 + bundle/config/resources/external_location.go | 2 +- .../resources/external_location_test.go | 2 +- bundle/config/resources_test.go | 10 ++++- bundle/direct/dresources/all_test.go | 20 ++++++++++ bundle/direct/dresources/resources.yml | 4 ++ bundle/direct/dresources/type_test.go | 3 ++ .../schema/annotations_openapi_overrides.yml | 32 ++++++++++++++++ bundle/schema/jsonschema.json | 37 ++++++++++++++++++- libs/structs/structwalk/walktype_test.go | 2 +- 12 files changed, 113 insertions(+), 7 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index ed0186787c..609215b053 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -20,6 +20,7 @@ import ( // These resources are there because they use grants, not permissions: var unsupportedResources = []string{ "catalogs", + "external_locations", "volumes", "schemas", "quality_monitors", diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 2b84eef3c9..d4e5123125 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -135,6 +135,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Catalogs: map[string]*resources.Catalog{ "catalog1": {CreateCatalog: catalog.CreateCatalog{Name: "catalog1"}}, }, + ExternalLocations: map[string]*resources.ExternalLocation{ + "externalLocation1": {CreateExternalLocation: catalog.CreateExternalLocation{Name: "externalLocation1"}}, + }, Schemas: map[string]*resources.Schema{ "schema1": {CreateSchema: catalog.CreateSchema{Name: "schema1"}}, }, @@ -405,10 +408,11 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) { b := mockBundle(config.Development) // UC resources should not have a prefix added to their name. Right now - // this list only contains the Volume and Catalog resources since we have yet to remove + // this list only contains the Volume, Catalog, and ExternalLocation resources since we have yet to remove // prefixing support for UC schemas and registered models. ucFields := []reflect.Type{ reflect.TypeOf(&resources.Catalog{}), + reflect.TypeOf(&resources.ExternalLocation{}), reflect.TypeOf(&resources.Volume{}), } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 3c62af9341..7cc46bfe8c 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -41,6 +41,7 @@ func allResourceTypes(t *testing.T) []string { "database_catalogs", "database_instances", "experiments", + "external_locations", "jobs", "model_serving_endpoints", "models", diff --git a/bundle/config/resources/external_location.go b/bundle/config/resources/external_location.go index f7a79f0d05..a44e9d7707 100644 --- a/bundle/config/resources/external_location.go +++ b/bundle/config/resources/external_location.go @@ -78,7 +78,7 @@ func (e *ExternalLocation) Exists(ctx context.Context, w *databricks.WorkspaceCl func (*ExternalLocation) ResourceDescription() ResourceDescription { return ResourceDescription{ - SingularName: "external location", + SingularName: "external_location", PluralName: "external_locations", SingularTitle: "External Location", PluralTitle: "External Locations", diff --git a/bundle/config/resources/external_location_test.go b/bundle/config/resources/external_location_test.go index 69f1ca2af2..76f82736ad 100644 --- a/bundle/config/resources/external_location_test.go +++ b/bundle/config/resources/external_location_test.go @@ -48,7 +48,7 @@ func TestExternalLocationResourceDescription(t *testing.T) { el := &ExternalLocation{} desc := el.ResourceDescription() - assert.Equal(t, "external location", desc.SingularName) + assert.Equal(t, "external_location", desc.SingularName) assert.Equal(t, "external_locations", desc.PluralName) assert.Equal(t, "External Location", desc.SingularTitle) assert.Equal(t, "External Locations", desc.PluralTitle) diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 373d696f62..b8dd140617 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -145,6 +145,11 @@ func TestResourcesBindSupport(t *testing.T) { CreateCatalog: catalog.CreateCatalog{}, }, }, + ExternalLocations: map[string]*resources.ExternalLocation{ + "my_external_location": { + CreateExternalLocation: catalog.CreateExternalLocation{}, + }, + }, Schemas: map[string]*resources.Schema{ "my_schema": { CreateSchema: catalog.CreateSchema{}, @@ -236,7 +241,9 @@ func TestResourcesBindSupport(t *testing.T) { }, }, } - unbindableResources := map[string]bool{"model": true} + unbindableResources := map[string]bool{ + "model": true, + } ctx := context.Background() m := mocks.NewMockWorkspaceClient(t) @@ -245,6 +252,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockExperimentsAPI().EXPECT().GetExperiment(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockRegisteredModelsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockExternalLocationsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 5233ecff47..29e8ebfc3c 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -49,6 +49,15 @@ var testConfig map[string]any = map[string]any{ // only during updates. They are not included in the test config. }, + "external_locations": &resources.ExternalLocation{ + CreateExternalLocation: catalog.CreateExternalLocation{ + Name: "myexternallocation", + Url: "s3://mybucket/mypath", + CredentialName: "mycredential", + Comment: "Test external location", + }, + }, + "schemas": &resources.Schema{ CreateSchema: catalog.CreateSchema{ CatalogName: "main", @@ -486,6 +495,17 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "external_locations.grants": func(client *databricks.WorkspaceClient) (any, error) { + return &GrantsState{ + SecurableType: "external_location", + FullName: "myexternallocation", + Grants: []GrantAssignment{{ + Privileges: []catalog.Privilege{catalog.PrivilegeReadFiles}, + Principal: "user@example.com", + }}, + }, nil + }, + "schemas.grants": func(client *databricks.WorkspaceClient) (any, error) { return &GrantsState{ SecurableType: "schema", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index c49f07de63..8d1449e8dc 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -141,6 +141,10 @@ resources: update_id_on_changes: - field: name reason: id_changes + ignore_remote_changes: + # skip_validation is input-only and not returned by the API + - field: skip_validation + reason: input_only volumes: recreate_on_changes: diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 6f237f415b..643f27423e 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,6 +19,9 @@ var knownMissingInRemoteType = map[string][]string{ "clusters": { "apply_policy_default_values", }, + "external_locations": { + "skip_validation", + }, "model_serving_endpoints": { "ai_gateway", "budget_policy_id", diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index f53ae72ebc..0048849901 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -1151,3 +1151,35 @@ github.com/databricks/databricks-sdk-go/service/catalog.SseEncryptionDetails: "aws_kms_key_arn": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/catalog.SseEncryptionDetailsAlgorithm: + "_": + "description": |- + SSE algorithm to use for encrypting S3 objects + "enum": + - |- + AWS_SSE_KMS + - |- + AWS_SSE_S3 +github.com/databricks/cli/bundle/config/resources.ExternalLocationGrantPrivilege: + "_": + "description": |- + Privilege to grant on an external location + "enum": + - |- + ALL_PRIVILEGES + - |- + CREATE_EXTERNAL_TABLE + - |- + CREATE_EXTERNAL_VOLUME + - |- + CREATE_MANAGED_STORAGE + - |- + CREATE_TABLE + - |- + CREATE_VOLUME + - |- + MANAGE + - |- + READ_FILES + - |- + WRITE_FILES diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index f898174a56..17de9f4842 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -976,7 +976,27 @@ ] }, "resources.ExternalLocationGrantPrivilege": { - "type": "string" + "oneOf": [ + { + "type": "string", + "description": "Privilege to grant on an external location", + "enum": [ + "ALL_PRIVILEGES", + "CREATE_EXTERNAL_TABLE", + "CREATE_EXTERNAL_VOLUME", + "CREATE_MANAGED_STORAGE", + "CREATE_TABLE", + "CREATE_VOLUME", + "MANAGE", + "READ_FILES", + "WRITE_FILES" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] }, "resources.Grant": { "oneOf": [ @@ -4274,7 +4294,20 @@ ] }, "catalog.SseEncryptionDetailsAlgorithm": { - "type": "string" + "oneOf": [ + { + "type": "string", + "description": "SSE algorithm to use for encrypting S3 objects", + "enum": [ + "AWS_SSE_KMS", + "AWS_SSE_S3" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] }, "catalog.VolumeType": { "oneOf": [ diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index 8743961d3c..5d64222bcc 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -136,7 +136,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeOf(config.Root{}), - 4300, 4750, // 4739 after adding external locations support + 4300, 4800, // 4754 after adding external locations support map[string]any{ "bundle.target": "", `variables.*.lookup.dashboard`: "", From 0402e7b7033c182ca793c109e9ebcd1d73dd8f27 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Feb 2026 13:47:10 +0000 Subject: [PATCH 4/8] updated refschema --- acceptance/bundle/refschema/out.fields.txt | 60 ++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 12e8da5992..a1fd833cdd 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -662,6 +662,66 @@ resources.experiments.*.permissions.permissions[*].group_name string ALL resources.experiments.*.permissions.permissions[*].permission_level iam.PermissionLevel ALL resources.experiments.*.permissions.permissions[*].service_principal_name string ALL resources.experiments.*.permissions.permissions[*].user_name string ALL +resources.external_locations.*.browse_only bool REMOTE +resources.external_locations.*.comment string ALL +resources.external_locations.*.created_at int64 REMOTE +resources.external_locations.*.created_by string REMOTE +resources.external_locations.*.credential_id string REMOTE +resources.external_locations.*.credential_name string ALL +resources.external_locations.*.enable_file_events bool ALL +resources.external_locations.*.encryption_details *catalog.EncryptionDetails ALL +resources.external_locations.*.encryption_details.sse_encryption_details *catalog.SseEncryptionDetails ALL +resources.external_locations.*.encryption_details.sse_encryption_details.algorithm catalog.SseEncryptionDetailsAlgorithm ALL +resources.external_locations.*.encryption_details.sse_encryption_details.aws_kms_key_arn string ALL +resources.external_locations.*.fallback bool ALL +resources.external_locations.*.file_event_queue *catalog.FileEventQueue ALL +resources.external_locations.*.file_event_queue.managed_aqs *catalog.AzureQueueStorage ALL +resources.external_locations.*.file_event_queue.managed_aqs.managed_resource_id string ALL +resources.external_locations.*.file_event_queue.managed_aqs.queue_url string ALL +resources.external_locations.*.file_event_queue.managed_aqs.resource_group string ALL +resources.external_locations.*.file_event_queue.managed_aqs.subscription_id string ALL +resources.external_locations.*.file_event_queue.managed_pubsub *catalog.GcpPubsub ALL +resources.external_locations.*.file_event_queue.managed_pubsub.managed_resource_id string ALL +resources.external_locations.*.file_event_queue.managed_pubsub.subscription_name string ALL +resources.external_locations.*.file_event_queue.managed_sqs *catalog.AwsSqsQueue ALL +resources.external_locations.*.file_event_queue.managed_sqs.managed_resource_id string ALL +resources.external_locations.*.file_event_queue.managed_sqs.queue_url string ALL +resources.external_locations.*.file_event_queue.provided_aqs *catalog.AzureQueueStorage ALL +resources.external_locations.*.file_event_queue.provided_aqs.managed_resource_id string ALL +resources.external_locations.*.file_event_queue.provided_aqs.queue_url string ALL +resources.external_locations.*.file_event_queue.provided_aqs.resource_group string ALL +resources.external_locations.*.file_event_queue.provided_aqs.subscription_id string ALL +resources.external_locations.*.file_event_queue.provided_pubsub *catalog.GcpPubsub ALL +resources.external_locations.*.file_event_queue.provided_pubsub.managed_resource_id string ALL +resources.external_locations.*.file_event_queue.provided_pubsub.subscription_name string ALL +resources.external_locations.*.file_event_queue.provided_sqs *catalog.AwsSqsQueue ALL +resources.external_locations.*.file_event_queue.provided_sqs.managed_resource_id string ALL +resources.external_locations.*.file_event_queue.provided_sqs.queue_url string ALL +resources.external_locations.*.grants []resources.ExternalLocationGrant INPUT +resources.external_locations.*.grants[*] resources.ExternalLocationGrant INPUT +resources.external_locations.*.grants[*].principal string INPUT +resources.external_locations.*.grants[*].privileges []resources.ExternalLocationGrantPrivilege INPUT +resources.external_locations.*.grants[*].privileges[*] resources.ExternalLocationGrantPrivilege INPUT +resources.external_locations.*.id string INPUT +resources.external_locations.*.isolation_mode catalog.IsolationMode REMOTE +resources.external_locations.*.lifecycle resources.Lifecycle INPUT +resources.external_locations.*.lifecycle.prevent_destroy bool INPUT +resources.external_locations.*.metastore_id string REMOTE +resources.external_locations.*.modified_status string INPUT +resources.external_locations.*.name string ALL +resources.external_locations.*.owner string REMOTE +resources.external_locations.*.read_only bool ALL +resources.external_locations.*.skip_validation bool INPUT STATE +resources.external_locations.*.updated_at int64 REMOTE +resources.external_locations.*.updated_by string REMOTE +resources.external_locations.*.url string ALL +resources.external_locations.*.grants.full_name string ALL +resources.external_locations.*.grants.grants []dresources.GrantAssignment ALL +resources.external_locations.*.grants.grants[*] dresources.GrantAssignment ALL +resources.external_locations.*.grants.grants[*].principal string ALL +resources.external_locations.*.grants.grants[*].privileges []catalog.Privilege ALL +resources.external_locations.*.grants.grants[*].privileges[*] catalog.Privilege ALL +resources.external_locations.*.grants.securable_type string ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL From 66fb34dd83b631baf578c9e15e1fdb84e1859b66 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Feb 2026 20:12:26 +0000 Subject: [PATCH 5/8] fixes --- bundle/direct/dresources/external_location.go | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/bundle/direct/dresources/external_location.go b/bundle/direct/dresources/external_location.go index e7d9def439..35d0267faa 100644 --- a/bundle/direct/dresources/external_location.go +++ b/bundle/direct/dresources/external_location.go @@ -31,9 +31,9 @@ func (*ResourceExternalLocation) RemapState(info *catalog.ExternalLocationInfo) FileEventQueue: info.FileEventQueue, Name: info.Name, ReadOnly: info.ReadOnly, - // Note: SkipValidation is input-only and not included in state - Url: info.Url, - ForceSendFields: utils.FilterFields[catalog.CreateExternalLocation](info.ForceSendFields), + SkipValidation: false, // This is an input-only parameter, never returned by API + Url: info.Url, + ForceSendFields: utils.FilterFields[catalog.CreateExternalLocation](info.ForceSendFields), } } @@ -52,21 +52,21 @@ func (r *ResourceExternalLocation) DoCreate(ctx context.Context, config *catalog // DoUpdate updates the external location in place and returns remote state. func (r *ResourceExternalLocation) DoUpdate(ctx context.Context, id string, config *catalog.CreateExternalLocation, _ Changes) (*catalog.ExternalLocationInfo, error) { updateRequest := catalog.UpdateExternalLocation{ - Comment: config.Comment, - CredentialName: config.CredentialName, - EnableFileEvents: config.EnableFileEvents, - // EncryptionDetails is not supported for updates - Fallback: config.Fallback, - // FileEventQueue is not supported for updates - Force: false, - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Only set if name actually changes (see DoUpdateWithID) - Owner: "", // Not supported by DABs - ReadOnly: config.ReadOnly, - SkipValidation: config.SkipValidation, - Url: config.Url, - ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](config.ForceSendFields, "IsolationMode", "Owner"), + Comment: config.Comment, + CredentialName: config.CredentialName, + EnableFileEvents: config.EnableFileEvents, + EncryptionDetails: config.EncryptionDetails, + Fallback: config.Fallback, + FileEventQueue: config.FileEventQueue, + Force: false, + IsolationMode: "", // Not supported by DABs + Name: id, + NewName: "", // Only set if name actually changes (see DoUpdateWithID) + Owner: "", // Not supported by DABs + ReadOnly: config.ReadOnly, + SkipValidation: config.SkipValidation, + Url: config.Url, + ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](config.ForceSendFields, "IsolationMode", "Owner"), } response, err := r.client.ExternalLocations.Update(ctx, updateRequest) From 4c26bb6c5fb099c6aee00f1a49de72832c45bf13 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 11 Feb 2026 13:11:15 +0000 Subject: [PATCH 6/8] addressed feedback --- .../databricks.yml.tmpl | 0 .../out.test.toml | 0 .../output.txt | 0 .../script | 0 .../test.toml | 0 bundle/direct/dresources/external_location.go | 37 ++++++++----------- libs/testserver/external_locations.go | 32 ---------------- libs/testserver/handlers.go | 4 +- 8 files changed, 18 insertions(+), 55 deletions(-) rename acceptance/bundle/resources/{catalogs_and_external_locations => external_locations}/databricks.yml.tmpl (100%) rename acceptance/bundle/resources/{catalogs_and_external_locations => external_locations}/out.test.toml (100%) rename acceptance/bundle/resources/{catalogs_and_external_locations => external_locations}/output.txt (100%) rename acceptance/bundle/resources/{catalogs_and_external_locations => external_locations}/script (100%) rename acceptance/bundle/resources/{catalogs_and_external_locations => external_locations}/test.toml (100%) diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/databricks.yml.tmpl b/acceptance/bundle/resources/external_locations/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/resources/catalogs_and_external_locations/databricks.yml.tmpl rename to acceptance/bundle/resources/external_locations/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/out.test.toml b/acceptance/bundle/resources/external_locations/out.test.toml similarity index 100% rename from acceptance/bundle/resources/catalogs_and_external_locations/out.test.toml rename to acceptance/bundle/resources/external_locations/out.test.toml diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/output.txt b/acceptance/bundle/resources/external_locations/output.txt similarity index 100% rename from acceptance/bundle/resources/catalogs_and_external_locations/output.txt rename to acceptance/bundle/resources/external_locations/output.txt diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/script b/acceptance/bundle/resources/external_locations/script similarity index 100% rename from acceptance/bundle/resources/catalogs_and_external_locations/script rename to acceptance/bundle/resources/external_locations/script diff --git a/acceptance/bundle/resources/catalogs_and_external_locations/test.toml b/acceptance/bundle/resources/external_locations/test.toml similarity index 100% rename from acceptance/bundle/resources/catalogs_and_external_locations/test.toml rename to acceptance/bundle/resources/external_locations/test.toml diff --git a/bundle/direct/dresources/external_location.go b/bundle/direct/dresources/external_location.go index 35d0267faa..2d0591e180 100644 --- a/bundle/direct/dresources/external_location.go +++ b/bundle/direct/dresources/external_location.go @@ -69,32 +69,27 @@ func (r *ResourceExternalLocation) DoUpdate(ctx context.Context, id string, conf ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](config.ForceSendFields, "IsolationMode", "Owner"), } - response, err := r.client.ExternalLocations.Update(ctx, updateRequest) - if err != nil { - return nil, err - } - - return response, nil + return r.client.ExternalLocations.Update(ctx, updateRequest) } // DoUpdateWithID updates the external location and returns the new ID if the name changes. func (r *ResourceExternalLocation) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateExternalLocation) (string, *catalog.ExternalLocationInfo, error) { updateRequest := catalog.UpdateExternalLocation{ - Comment: config.Comment, - CredentialName: config.CredentialName, - EnableFileEvents: config.EnableFileEvents, - // EncryptionDetails is not supported for updates - Fallback: config.Fallback, - // FileEventQueue is not supported for updates - Force: false, - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Initialized below if needed - Owner: "", // Not supported by DABs - ReadOnly: config.ReadOnly, - SkipValidation: config.SkipValidation, - Url: config.Url, - ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](config.ForceSendFields, "IsolationMode", "Owner"), + Comment: config.Comment, + CredentialName: config.CredentialName, + EnableFileEvents: config.EnableFileEvents, + EncryptionDetails: config.EncryptionDetails, + Fallback: config.Fallback, + FileEventQueue: config.FileEventQueue, + Force: false, + IsolationMode: "", // Not supported by DABs + Name: id, + NewName: "", // Initialized below if needed + Owner: "", // Not supported by DABs + ReadOnly: config.ReadOnly, + SkipValidation: config.SkipValidation, + Url: config.Url, + ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](config.ForceSendFields, "IsolationMode", "Owner"), } if config.Name != id { diff --git a/libs/testserver/external_locations.go b/libs/testserver/external_locations.go index aed92c6644..0606ad76e5 100644 --- a/libs/testserver/external_locations.go +++ b/libs/testserver/external_locations.go @@ -103,35 +103,3 @@ func (s *FakeWorkspace) ExternalLocationsUpdate(req Request, name string) Respon Body: existing, } } - -func (s *FakeWorkspace) ExternalLocationsGet(_ Request, name string) Response { - defer s.LockUnlock()() - - existing, ok := s.ExternalLocations[name] - if !ok { - return Response{ - StatusCode: http.StatusNotFound, - Body: fmt.Sprintf("external location %s not found", name), - } - } - - return Response{ - Body: existing, - } -} - -func (s *FakeWorkspace) ExternalLocationsDelete(_ Request, name string) Response { - defer s.LockUnlock()() - - if _, ok := s.ExternalLocations[name]; !ok { - return Response{ - StatusCode: http.StatusNotFound, - Body: fmt.Sprintf("external location %s not found", name), - } - } - - delete(s.ExternalLocations, name) - return Response{ - StatusCode: http.StatusOK, - } -} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 15c720f508..fcda716bc3 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -436,7 +436,7 @@ func AddDefaultHandlers(server *Server) { // External Locations: server.Handle("GET", "/api/2.1/unity-catalog/external-locations/{name}", func(req Request) any { - return req.Workspace.ExternalLocationsGet(req, req.Vars["name"]) + return MapGet(req.Workspace, req.Workspace.ExternalLocations, req.Vars["name"]) }) server.Handle("POST", "/api/2.1/unity-catalog/external-locations", func(req Request) any { @@ -448,7 +448,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("DELETE", "/api/2.1/unity-catalog/external-locations/{name}", func(req Request) any { - return req.Workspace.ExternalLocationsDelete(req, req.Vars["name"]) + return MapDelete(req.Workspace, req.Workspace.ExternalLocations, req.Vars["name"]) }) // Registered Models: From 888cd00e5a4fa0c1cf4a0956056baa7ffed979a2 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 11 Feb 2026 13:13:37 +0000 Subject: [PATCH 7/8] added invariant test --- .../invariant/configs/external_location.yml.tmpl | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 acceptance/bundle/invariant/configs/external_location.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/external_location.yml.tmpl b/acceptance/bundle/invariant/configs/external_location.yml.tmpl new file mode 100644 index 0000000000..500cf2d41d --- /dev/null +++ b/acceptance/bundle/invariant/configs/external_location.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + external_locations: + test_location: + name: test_location_$UNIQUE_NAME + url: s3://test-bucket/path + credential_name: test_storage_credential + comment: "Test external location from DABs" From 0fd9a36625d592f5a13607b474453040564f11b2 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 11 Feb 2026 13:14:24 +0000 Subject: [PATCH 8/8] changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 2cf027cb00..ce1e641950 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI ### Bundles +* Added support for UC external locations (direct mode only) ([#4484](https://github.com/databricks/cli/pull/4484)) ### Dependency updates