Skip to content

Commit

Permalink
[Dependency] Bump Go SDK to v0.50.0 (#4178)
Browse files Browse the repository at this point in the history 
## Changes
Use the latest Go SDK in the Terraform provider. The main changes affect
Dashboards and Online Tables, whose Create and Update RPCs now accept an
instance of the resource, rather than inlining the fields of the
resource. Additionally, Online Tables introduced a waiter configuration,
so we can remove hand-written waiter logic used before.

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [ ] `make test` run locally
- [ ] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] relevant acceptance tests are passing
- [ ] using Go SDK
  • Loading branch information
@mgyucht
mgyucht authored Oct 31, 2024
1 parent 5daf2ed commit 17641de
Show file tree
Hide file tree
Showing 15 changed files with 395 additions and 419 deletions.
2 changes: 1 addition & 1 deletion .codegen/_openapi_sha
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
cf9c61453990df0f9453670f2fe68e1b128647a2
25b2478e5a18c888f0d423249abde5499dc58424
31 changes: 4 additions & 27 deletions catalog/resource_online_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,6 @@ import (

const onlineTableDefaultProvisionTimeout = 90 * time.Minute

func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error {
return retry.RetryContext(ctx, onlineTableDefaultProvisionTimeout, func() *retry.RetryError {
endpoint, err := w.OnlineTables.GetByName(ctx, onlineTableName)
if err != nil {
return retry.NonRetryableError(err)
}
if endpoint.Status == nil {
return retry.RetryableError(fmt.Errorf("online table status is not available yet"))
}
switch endpoint.Status.DetailedState {
case catalog.OnlineTableStateOnline, catalog.OnlineTableStateOnlineContinuousUpdate,
catalog.OnlineTableStateOnlineNoPendingUpdate, catalog.OnlineTableStateOnlineTriggeredUpdate:
return nil

// does catalog.OnlineTableStateOffline means that it's failed?
case catalog.OnlineTableStateOfflineFailed, catalog.OnlineTableStateOnlinePipelineFailed:
return retry.NonRetryableError(fmt.Errorf("online table status returned %s for online table: %s",
endpoint.Status.DetailedState.String(), onlineTableName))
}
return retry.RetryableError(fmt.Errorf("online table %s is still pending", onlineTableName))
})
}

func waitForOnlineTableDeletion(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error {
return retry.RetryContext(ctx, onlineTableDefaultProvisionTimeout, func() *retry.RetryError {
_, err := w.OnlineTables.GetByName(ctx, onlineTableName)
Expand Down Expand Up @@ -75,17 +52,17 @@ func ResourceOnlineTable() common.Resource {
if err != nil {
return err
}
var req catalog.CreateOnlineTableRequest
common.DataToStructPointer(d, s, &req)
res, err := w.OnlineTables.Create(ctx, req)
var table catalog.OnlineTable
common.DataToStructPointer(d, s, &table)
res, err := w.OnlineTables.Create(ctx, catalog.CreateOnlineTableRequest{Table: &table})
if err != nil {
return err
}
// Note: We should set the id right after creation and before waiting for online table to be available.
// If the resource creation timeout is exceeded while waiting for the online table to be ready, this ensures the online table is persisted in the state.
d.SetId(res.Name)
// this should be specified in the API Spec - filed a ticket to add it
err = waitForOnlineTableCreation(w, ctx, res.Name)
_, err = res.GetWithTimeout(onlineTableDefaultProvisionTimeout)
if err != nil {
return err
}
Expand Down
66 changes: 42 additions & 24 deletions catalog/resource_online_table_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package catalog

import (
"errors"
"fmt"
"testing"
"time"

"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
Expand Down Expand Up @@ -47,6 +49,13 @@ func TestOnlineTableCreate(t *testing.T) {
PrimaryKeyColumns: []string{"id"},
},
}
otStatusNotSetWait := &catalog.WaitGetOnlineTableActive[catalog.OnlineTable]{
Response: otStatusNotSet,
Name: "main.default.online_table",
Poll: func(d time.Duration, f func(*catalog.OnlineTable)) (*catalog.OnlineTable, error) {
return otStatusOnline, nil
},
}
// otStatusUnknown := &catalog.OnlineTable{
// Name: "main.default.online_table",
// Spec: &catalog.OnlineTableSpec{
Expand All @@ -60,16 +69,15 @@ func TestOnlineTableCreate(t *testing.T) {
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockOnlineTablesAPI().EXPECT()
e.Create(mock.Anything, catalog.CreateOnlineTableRequest{
Name: "main.default.online_table",
Spec: &catalog.OnlineTableSpec{
RunTriggered: &catalog.OnlineTableSpecTriggeredSchedulingPolicy{},
SourceTableFullName: "main.default.test",
PrimaryKeyColumns: []string{"id"},
Table: &catalog.OnlineTable{
Name: "main.default.online_table",
Spec: &catalog.OnlineTableSpec{
RunTriggered: &catalog.OnlineTableSpecTriggeredSchedulingPolicy{},
SourceTableFullName: "main.default.test",
PrimaryKeyColumns: []string{"id"},
},
},
}).Return(otStatusNotSet, nil)
// TODO: how to emulate the status change
// e.GetByName(mock.Anything, "main.default.online_table").Return(otStatusNotSet, nil)
// e.GetByName(mock.Anything, "main.default.online_table").Return(otStatusUnknown, nil)
}).Return(otStatusNotSetWait, nil)
e.GetByName(mock.Anything, "main.default.online_table").Return(otStatusOnline, nil)
},
Resource: ResourceOnlineTable(),
Expand All @@ -85,11 +93,13 @@ func TestOnlineTableCreate_ErrorImmediately(t *testing.T) {
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockOnlineTablesAPI().EXPECT()
e.Create(mock.Anything, catalog.CreateOnlineTableRequest{
Name: "main.default.online_table",
Spec: &catalog.OnlineTableSpec{
RunTriggered: &catalog.OnlineTableSpecTriggeredSchedulingPolicy{},
SourceTableFullName: "main.default.test",
PrimaryKeyColumns: []string{"id"},
Table: &catalog.OnlineTable{
Name: "main.default.online_table",
Spec: &catalog.OnlineTableSpec{
RunTriggered: &catalog.OnlineTableSpecTriggeredSchedulingPolicy{},
SourceTableFullName: "main.default.test",
PrimaryKeyColumns: []string{"id"},
},
},
}).Return(nil, fmt.Errorf("error!"))
},
Expand All @@ -100,33 +110,41 @@ func TestOnlineTableCreate_ErrorImmediately(t *testing.T) {
}

func TestOnlineTableCreate_ErrorInWait(t *testing.T) {
otStatusError := &catalog.OnlineTable{
otStatusProvisioning := &catalog.OnlineTable{
Name: "main.default.online_table",
Spec: &catalog.OnlineTableSpec{
RunTriggered: &catalog.OnlineTableSpecTriggeredSchedulingPolicy{},
SourceTableFullName: "main.default.test",
PrimaryKeyColumns: []string{"id"},
},
Status: &catalog.OnlineTableStatus{DetailedState: catalog.OnlineTableStateOfflineFailed},
Status: &catalog.OnlineTableStatus{DetailedState: catalog.OnlineTableStateProvisioning},
}
otStatusErrorWait := &catalog.WaitGetOnlineTableActive[catalog.OnlineTable]{
Response: otStatusProvisioning,
Name: "main.default.online_table",
Poll: func(d time.Duration, f func(*catalog.OnlineTable)) (*catalog.OnlineTable, error) {
return nil, errors.New("failed to reach ACTIVE, got OFFLINE_FAILED: error!")
},
}
d, err := qa.ResourceFixture{
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockOnlineTablesAPI().EXPECT()
e.Create(mock.Anything, catalog.CreateOnlineTableRequest{
Name: "main.default.online_table",
Spec: &catalog.OnlineTableSpec{
RunTriggered: &catalog.OnlineTableSpecTriggeredSchedulingPolicy{},
SourceTableFullName: "main.default.test",
PrimaryKeyColumns: []string{"id"},
Table: &catalog.OnlineTable{
Name: "main.default.online_table",
Spec: &catalog.OnlineTableSpec{
RunTriggered: &catalog.OnlineTableSpecTriggeredSchedulingPolicy{},
SourceTableFullName: "main.default.test",
PrimaryKeyColumns: []string{"id"},
},
},
}).Return(otStatusError, nil)
e.GetByName(mock.Anything, "main.default.online_table").Return(otStatusError, nil)
}).Return(otStatusErrorWait, nil)
},
Resource: ResourceOnlineTable(),
HCL: onlineTableHcl,
Create: true,
}.Apply(t)
qa.AssertErrorStartsWith(t, err, "online table status returned OFFLINE_FAILED for online table: main.default.online_table")
qa.AssertErrorStartsWith(t, err, "failed to reach ACTIVE, got OFFLINE_FAILED: error!")
assert.Equal(t, "main.default.online_table", d.Id())
}

Expand Down
27 changes: 15 additions & 12 deletions dashboards/resource_dashboard.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,22 +68,22 @@ func ResourceDashboard() common.Resource {
if err != nil {
return err
}
var newDashboardRequest dashboards.CreateDashboardRequest
common.DataToStructPointer(d, dashboardSchema, &newDashboardRequest)
var dashboard dashboards.Dashboard
common.DataToStructPointer(d, dashboardSchema, &dashboard)
content, md5Hash, err := common.ReadSerializedJsonContent(d.Get("serialized_dashboard").(string), d.Get("file_path").(string))
if err != nil {
return err
}
d.Set("md5", md5Hash)
newDashboardRequest.SerializedDashboard = content
createdDashboard, err := w.Lakeview.Create(ctx, newDashboardRequest)
dashboard.SerializedDashboard = content
createdDashboard, err := w.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{Dashboard: &dashboard})
if err != nil && isParentDoesntExistError(err) {
log.Printf("[DEBUG] Parent folder '%s' doesn't exist, creating...", newDashboardRequest.ParentPath)
err = w.Workspace.MkdirsByPath(ctx, newDashboardRequest.ParentPath)
log.Printf("[DEBUG] Parent folder '%s' doesn't exist, creating...", dashboard.ParentPath)
err = w.Workspace.MkdirsByPath(ctx, dashboard.ParentPath)
if err != nil {
return err
}
createdDashboard, err = w.Lakeview.Create(ctx, newDashboardRequest)
createdDashboard, err = w.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{Dashboard: &dashboard})
}
if err != nil {
return err
Expand Down Expand Up @@ -132,16 +132,19 @@ func ResourceDashboard() common.Resource {
if err != nil {
return err
}
var updateDashboardRequest dashboards.UpdateDashboardRequest
common.DataToStructPointer(d, dashboardSchema, &updateDashboardRequest)
updateDashboardRequest.DashboardId = d.Id()
var dashboard dashboards.Dashboard
common.DataToStructPointer(d, dashboardSchema, &dashboard)
dashboard.DashboardId = d.Id()
content, md5Hash, err := common.ReadSerializedJsonContent(d.Get("serialized_dashboard").(string), d.Get("file_path").(string))
if err != nil {
return err
}
d.Set("md5", md5Hash)
updateDashboardRequest.SerializedDashboard = content
updatedDashboard, err := w.Lakeview.Update(ctx, updateDashboardRequest)
dashboard.SerializedDashboard = content
updatedDashboard, err := w.Lakeview.Update(ctx, dashboards.UpdateDashboardRequest{
DashboardId: dashboard.DashboardId,
Dashboard: &dashboard,
})
if err != nil {
return err
}
Expand Down
42 changes: 26 additions & 16 deletions dashboards/resource_dashboard_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@ func TestDashboardCreate(t *testing.T) {
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakeviewAPI().EXPECT()
e.Create(mock.Anything, dashboards.CreateDashboardRequest{
DisplayName: "Dashboard name",
WarehouseId: "abc",
ParentPath: "/path",
SerializedDashboard: "serialized_json",
Dashboard: &dashboards.Dashboard{
DisplayName: "Dashboard name",
WarehouseId: "abc",
ParentPath: "/path",
SerializedDashboard: "serialized_json",
},
}).Return(&dashboards.Dashboard{
DashboardId: "xyz",
DisplayName: "Dashboard name",
Expand Down Expand Up @@ -67,17 +69,21 @@ func TestDashboardCreate_NoParent(t *testing.T) {
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
lv := w.GetMockLakeviewAPI().EXPECT()
lv.Create(mock.Anything, dashboards.CreateDashboardRequest{
DisplayName: "Dashboard name",
WarehouseId: "abc",
ParentPath: "/path",
SerializedDashboard: "serialized_json",
Dashboard: &dashboards.Dashboard{
DisplayName: "Dashboard name",
WarehouseId: "abc",
ParentPath: "/path",
SerializedDashboard: "serialized_json",
},
}).Return(nil, fmt.Errorf("Path (/path) doesn't exist.")).Once()
w.GetMockWorkspaceAPI().EXPECT().MkdirsByPath(mock.Anything, "/path").Return(nil)
lv.Create(mock.Anything, dashboards.CreateDashboardRequest{
DisplayName: "Dashboard name",
WarehouseId: "abc",
ParentPath: "/path",
SerializedDashboard: "serialized_json",
Dashboard: &dashboards.Dashboard{
DisplayName: "Dashboard name",
WarehouseId: "abc",
ParentPath: "/path",
SerializedDashboard: "serialized_json",
},
}).Return(&dashboards.Dashboard{
DashboardId: "xyz",
DisplayName: "Dashboard name",
Expand Down Expand Up @@ -154,10 +160,14 @@ func TestDashboardUpdate(t *testing.T) {
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakeviewAPI().EXPECT()
e.Update(mock.Anything, dashboards.UpdateDashboardRequest{
DashboardId: "xyz",
DisplayName: "Dashboard name",
WarehouseId: "abc",
SerializedDashboard: "serialized_dashboard_updated",
DashboardId: "xyz",
Dashboard: &dashboards.Dashboard{
DashboardId: "xyz",
DisplayName: "Dashboard name",
WarehouseId: "abc",
SerializedDashboard: "serialized_dashboard_updated",
ParentPath: "/path",
},
}).Return(&dashboards.Dashboard{
DashboardId: "xyz",
DisplayName: "Dashboard name",
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/databricks/terraform-provider-databricks
go 1.22

require (
github.com/databricks/databricks-sdk-go v0.49.0
github.com/databricks/databricks-sdk-go v0.50.0
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320
github.com/hashicorp/hcl v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBS
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
github.com/databricks/databricks-sdk-go v0.49.0 h1:VBTeZZMLIuBSM4kxOCfUcW9z4FUQZY2QeNRD5qm9FUQ=
github.com/databricks/databricks-sdk-go v0.49.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU=
github.com/databricks/databricks-sdk-go v0.50.0 h1:Zl4uBhYMT5z6aDojCQJPT2zCYjjfqxBQSQn8uLTphpo=
github.com/databricks/databricks-sdk-go v0.50.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down
26 changes: 16 additions & 10 deletions internal/acceptance/dashboard_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,11 +315,14 @@ func TestAccDashboardWithRemoteChange(t *testing.T) {
w, err := databricks.NewWorkspaceClient(&databricks.Config{})
require.NoError(t, err)
_, err = w.Lakeview.Update(context.Background(), dashboards.UpdateDashboardRequest{
DashboardId: dashboard_id,
DisplayName: display_name,
Etag: etag,
WarehouseId: warehouse_id,
SerializedDashboard: "{\"pages\":[{\"name\":\"b532570b\",\"displayName\":\"New Page Modified Remote\"}]}",
DashboardId: dashboard_id,
Dashboard: &dashboards.Dashboard{
DashboardId: dashboard_id,
DisplayName: display_name,
Etag: etag,
WarehouseId: warehouse_id,
SerializedDashboard: "{\"pages\":[{\"name\":\"b532570b\",\"displayName\":\"New Page Modified Remote\"}]}",
},
})
require.NoError(t, err)
},
Expand Down Expand Up @@ -419,11 +422,14 @@ func TestAccDashboardTestAll(t *testing.T) {
w, err := databricks.NewWorkspaceClient(&databricks.Config{})
require.NoError(t, err)
_, err = w.Lakeview.Update(context.Background(), dashboards.UpdateDashboardRequest{
DashboardId: dashboard_id,
DisplayName: display_name,
Etag: etag,
WarehouseId: warehouse_id,
SerializedDashboard: "{\"pages\":[{\"name\":\"b532570b\",\"displayName\":\"New Page Modified Remote\"}]}",
DashboardId: dashboard_id,
Dashboard: &dashboards.Dashboard{
DashboardId: dashboard_id,
DisplayName: display_name,
Etag: etag,
WarehouseId: warehouse_id,
SerializedDashboard: "{\"pages\":[{\"name\":\"b532570b\",\"displayName\":\"New Page Modified Remote\"}]}",
},
})
require.NoError(t, err)
},
Expand Down
Loading

0 comments on commit 17641de

Please sign in to comment.