From 06495ba24ced143f9a0ebd300ad7a9f08e1184ab Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 28 Feb 2025 17:15:54 +0100 Subject: [PATCH 1/4] add support for schema's in deployment bind/unbind commands --- bundle/config/resources.go | 6 +++ bundle/config/resources/schema.go | 14 ++++-- integration/bundle/bind_resource_test.go | 54 ++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 1f523fed302..866bd764163 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -112,6 +112,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) } } + for k := range r.Schemas { + if k == key { + found = append(found, r.Schemas[k]) + } + } + if len(found) == 0 { return nil, fmt.Errorf("no such resource: %s", key) } diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go index b638907ace4..88bbfacb0d0 100644 --- a/bundle/config/resources/schema.go +++ b/bundle/config/resources/schema.go @@ -2,10 +2,11 @@ package resources import ( "context" - "errors" "net/url" "strings" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -25,8 +26,15 @@ type Schema struct { URL string `json:"url,omitempty" bundle:"internal"` } -func (s *Schema) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { - return false, errors.New("schema.Exists() is not supported") +func (s *Schema) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { + log.Tracef(ctx, "Checking if schema with fullName=%s exists", fullName) + + _, err := w.Schemas.GetByFullName(ctx, fullName) + if err != nil { + log.Debugf(ctx, "schema with full name %s does not exist", fullName) + return false, err + } + return true, nil } func (s *Schema) TerraformResourceName() string { diff --git a/integration/bundle/bind_resource_test.go b/integration/bundle/bind_resource_test.go index ba10190aaa3..89b0bbc3b8a 100644 --- a/integration/bundle/bind_resource_test.go +++ b/integration/bundle/bind_resource_test.go @@ -15,8 +15,62 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/databricks/databricks-sdk-go/service/catalog" ) +func TestBindSchemaToExistingSchema(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + + // create a pre-defined schema: + uniqueId := uuid.New().String() + predefinedSchema, err := wt.W.Schemas.Create(ctx, catalog.CreateSchema{ + CatalogName: "main", + Name: "test-schema-" + uniqueId, + }) + require.NoError(t, err) + t.Cleanup(func() { + err := wt.W.Schemas.DeleteByFullName(ctx, predefinedSchema.FullName) + require.NoError(t, err) + }) + + // setup the bundle: + bundleRoot := initTestTemplate(t, ctx, "uc_schema", map[string]any{ + "unique_id": uniqueId, + }) + ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot) + + // run the bind command: + c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "bar", predefinedSchema.FullName, "--auto-approve") + _, _, err = c.Run() + require.NoError(t, err) + + // deploy the bundle: + deployBundle(t, ctx, bundleRoot) + + // Check that predefinedSchema is updated with config from bundle + w, err := databricks.NewWorkspaceClient() + require.NoError(t, err) + + updatedSchema, err := w.Schemas.GetByFullName(ctx, predefinedSchema.FullName) + require.NoError(t, err) + require.Equal(t, updatedSchema.SchemaId, predefinedSchema.SchemaId) + require.Equal(t, "This schema was created from DABs", updatedSchema.Comment) + + // unbind the schema: + c = testcli.NewRunner(t, ctx, "bundle", "deployment", "unbind", "bar") + _, _, err = c.Run() + require.NoError(t, err) + + // destroy the bundle: + destroyBundle(t, ctx, bundleRoot) + + // Check that schema is unbound and exists after bundle is destroyed + postDestroySchema, err := w.Schemas.GetByFullName(ctx, predefinedSchema.FullName) + require.NoError(t, err) + require.Equal(t, postDestroySchema.SchemaId, predefinedSchema.SchemaId) +} + func TestBindJobToExistingJob(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) gt := &generateJobTest{T: wt, w: wt.W} From 916d12a9ffd23f01c86d916d98695cb54b86207a Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:06:47 +0100 Subject: [PATCH 2/4] add a dedicated template for the integration test --- integration/bundle/bind_resource_test.go | 6 +++--- .../uc_schema_only/databricks_template_schema.json | 8 ++++++++ .../uc_schema_only/template/databricks.yml.tmpl | 13 +++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 integration/bundle/bundles/uc_schema_only/databricks_template_schema.json create mode 100644 integration/bundle/bundles/uc_schema_only/template/databricks.yml.tmpl diff --git a/integration/bundle/bind_resource_test.go b/integration/bundle/bind_resource_test.go index 89b0bbc3b8a..b182150ade1 100644 --- a/integration/bundle/bind_resource_test.go +++ b/integration/bundle/bind_resource_test.go @@ -35,13 +35,13 @@ func TestBindSchemaToExistingSchema(t *testing.T) { }) // setup the bundle: - bundleRoot := initTestTemplate(t, ctx, "uc_schema", map[string]any{ + bundleRoot := initTestTemplate(t, ctx, "uc_schema_only", map[string]any{ "unique_id": uniqueId, }) ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot) // run the bind command: - c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "bar", predefinedSchema.FullName, "--auto-approve") + c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "schema1", predefinedSchema.FullName, "--auto-approve") _, _, err = c.Run() require.NoError(t, err) @@ -58,7 +58,7 @@ func TestBindSchemaToExistingSchema(t *testing.T) { require.Equal(t, "This schema was created from DABs", updatedSchema.Comment) // unbind the schema: - c = testcli.NewRunner(t, ctx, "bundle", "deployment", "unbind", "bar") + c = testcli.NewRunner(t, ctx, "bundle", "deployment", "unbind", "schema1") _, _, err = c.Run() require.NoError(t, err) diff --git a/integration/bundle/bundles/uc_schema_only/databricks_template_schema.json b/integration/bundle/bundles/uc_schema_only/databricks_template_schema.json new file mode 100644 index 00000000000..1d3cdec85b3 --- /dev/null +++ b/integration/bundle/bundles/uc_schema_only/databricks_template_schema.json @@ -0,0 +1,8 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for the schema name" + } + } +} diff --git a/integration/bundle/bundles/uc_schema_only/template/databricks.yml.tmpl b/integration/bundle/bundles/uc_schema_only/template/databricks.yml.tmpl new file mode 100644 index 00000000000..39c64f12bc3 --- /dev/null +++ b/integration/bundle/bundles/uc_schema_only/template/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: uc-schema-only + +workspace: + root_path: "~/.bundle/{{.unique_id}}" + +resources: + schemas: + schema1: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created from DABs + From 63cfea2679b4c20e5f8b550dffa00045b42c8a1a Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:19:56 +0100 Subject: [PATCH 3/4] discern the schema not existing without an error --- bundle/config/resources/schema.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go index 88bbfacb0d0..d9849fd2d61 100644 --- a/bundle/config/resources/schema.go +++ b/bundle/config/resources/schema.go @@ -2,9 +2,12 @@ package resources import ( "context" + "errors" "net/url" "strings" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -31,7 +34,15 @@ func (s *Schema) Exists(ctx context.Context, w *databricks.WorkspaceClient, full _, err := w.Schemas.GetByFullName(ctx, fullName) if err != nil { - log.Debugf(ctx, "schema with full name %s does not exist", fullName) + log.Debugf(ctx, "schema with full name %s does not exist: %v", fullName, err) + + var aerr *apierr.APIError + if errors.As(err, &aerr) { + if aerr.StatusCode == 404 { + return false, nil + } + } + return false, err } return true, nil From a4600d3609003243883d6bcc05048a9ecd9c0b56 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:53:33 +0100 Subject: [PATCH 4/4] add unit test TestSchemaNotFound --- bundle/config/resources/schema_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 bundle/config/resources/schema_test.go diff --git a/bundle/config/resources/schema_test.go b/bundle/config/resources/schema_test.go new file mode 100644 index 00000000000..b609b65654c --- /dev/null +++ b/bundle/config/resources/schema_test.go @@ -0,0 +1,26 @@ +package resources + +import ( + "context" + "testing" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestSchemaNotFound(t *testing.T) { + ctx := context.Background() + + m := mocks.NewMockWorkspaceClient(t) + m.GetMockSchemasAPI().On("GetByFullName", mock.Anything, "non-existent-schema").Return(nil, &apierr.APIError{ + StatusCode: 404, + }) + + s := &Schema{} + exists, err := s.Exists(ctx, m.WorkspaceClient, "non-existent-schema") + + require.Falsef(t, exists, "Exists should return false when getting a 404 response from Workspace") + require.NoErrorf(t, err, "Exists should not return an error when getting a 404 response from Workspace") +}