Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Add support for setting table tags via terraform #4285

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 55 additions & 10 deletions catalog/resource_sql_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type SqlTableInfo struct {
ViewDefinition string `json:"view_definition,omitempty"`
Comment string `json:"comment,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
Tags map[string]string `json:"tags,omitempty"`
Options map[string]string `json:"options,omitempty" tf:"force_new"`
// EffectiveProperties includes both properties and options. Options are prefixed with `option.`.
EffectiveProperties map[string]string `json:"effective_properties" tf:"computed"`
Expand Down Expand Up @@ -243,19 +244,23 @@ func (ti *SqlTableInfo) serializeColumnInfos() string {
}

func (ti *SqlTableInfo) serializeProperties() string {
propsMap := make([]string, 0, len(ti.Properties))
for key, value := range ti.Properties {
propsMap = append(propsMap, fmt.Sprintf("'%s'='%s'", key, value))
}
return strings.Join(propsMap[:], ", ") // 'foo'='bar', 'this'='that'
return serializeMapField(ti.Properties)
}

func (ti *SqlTableInfo) serializeOptions() string {
optionsMap := make([]string, 0, len(ti.Options))
for key, value := range ti.Options {
optionsMap = append(optionsMap, fmt.Sprintf("'%s'='%s'", key, value))
return serializeMapField(ti.Options)
}

func (ti *SqlTableInfo) serializeTags() string {
return serializeMapField(ti.Tags)
}

func serializeMapField(field map[string]string) string {
propsMap := make([]string, 0, len(field))
for key, value := range field {
propsMap = append(propsMap, fmt.Sprintf("'%s'='%s'", key, value))
}
return strings.Join(optionsMap[:], ", ") // 'foo'='bar', 'this'='that'
return strings.Join(propsMap[:], ", ") // 'foo'='bar', 'this'='that'
}

func (ti *SqlTableInfo) buildLocationStatement() string {
Expand All @@ -276,7 +281,7 @@ func (ti *SqlTableInfo) getTableTypeString() string {
}

func (ti *SqlTableInfo) buildTableCreateStatement() string {
statements := make([]string, 0, 10)
statements := make([]string, 0, 11)

isView := ti.TableType == "VIEW"

Expand All @@ -289,6 +294,10 @@ func (ti *SqlTableInfo) buildTableCreateStatement() string {

statements = append(statements, fmt.Sprintf("CREATE %s%s %s", externalFragment, createType, ti.SQLFullName()))

if len(ti.Tags) > 0 {
statements = append(statements, fmt.Sprintf("ALTER %s %s SET TAGS (%s);", createType, ti.SQLFullName(), ti.serializeTags()))
}

if len(ti.ColumnInfos) > 0 {
statements = append(statements, fmt.Sprintf(" (%s)", ti.serializeColumnInfos()))
}
Expand Down Expand Up @@ -329,6 +338,10 @@ func (ti *SqlTableInfo) buildTableCreateStatement() string {

statements = append(statements, ";")

if len(ti.Tags) > 0 {
statements = append(statements, fmt.Sprintf("ALTER %s %s SET TAGS (%s);", createType, ti.SQLFullName(), ti.serializeTags()))
}

return strings.Join(statements, "")
}

Expand Down Expand Up @@ -453,6 +466,23 @@ func (ti *SqlTableInfo) diff(oldti *SqlTableInfo) ([]string, error) {
statements = append(statements, fmt.Sprintf("ALTER %s %s SET TBLPROPERTIES (%s)", typestring, ti.SQLFullName(), ti.serializeProperties()))
}

if !reflect.DeepEqual(ti.Tags, oldti.Tags) {
// First handle removal of tags
removeTags := make([]string, 0)
for key := range oldti.Tags {
if _, ok := ti.Tags[key]; !ok {
// These come from the terraform state rather than the api so they aren't escaped
removeTags = append(removeTags, fmt.Sprintf("'%s'", key))
}
}

if len(removeTags) > 0 {
statements = append(statements, fmt.Sprintf("ALTER %s %s UNSET TAGS (%s)", typestring, ti.SQLFullName(), strings.Join(removeTags, ",")))
}
// Next handle property changes and additions
statements = append(statements, fmt.Sprintf("ALTER %s %s SET TAGS (%s)", typestring, ti.SQLFullName(), ti.serializeTags()))
}

statements = ti.getStatementsForColumnDiffs(oldti, statements, typestring)

return statements, nil
Expand Down Expand Up @@ -600,6 +630,7 @@ func ResourceSqlTable() common.Resource {
// reset the effective property/option to the requested value.
userSpecifiedProperties := d.Get("properties").(map[string]any)
userSpecifiedOptions := d.Get("options").(map[string]any)
userSpecifiedTags := d.Get("tags").(map[string]any)
effectiveProperties := d.Get("effective_properties").(map[string]any)
diff := make(map[string]any)
for k, userSpecifiedValue := range userSpecifiedProperties {
Expand All @@ -625,6 +656,11 @@ func ResourceSqlTable() common.Resource {
diff[effectOptName] = userSpecifiedValue
}
}
for k, userSpecifiedValue := range userSpecifiedTags {
if effectiveValue, ok := effectiveProperties[k]; !ok || effectiveValue != userSpecifiedValue {
diff[k] = userSpecifiedValue
}
}
if len(diff) > 0 {
for k, v := range diff {
effectiveProperties[k] = v
Expand Down Expand Up @@ -688,6 +724,15 @@ func ResourceSqlTable() common.Resource {
return err
}
oldti, err := NewSqlTablesAPI(ctx, c).getTable(d.Id())
// The databricks API doesn't return tags with the table object so we use the terraform state
if d.HasChange("tags") && oldti.Tags == nil {
oldti.Tags = make(map[string]string)
old, _ := d.GetChange("tags")
oldProps := old.(map[string]any)
for key := range oldProps {
oldti.Tags[key] = oldProps[key].(string)
}
}
if err != nil {
return err
}
Expand Down
41 changes: 41 additions & 0 deletions catalog/resource_sql_table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,28 @@ func TestResourceSqlTableCreateStatement_Partition(t *testing.T) {
assert.Contains(t, stmt, "PARTITIONED BY (baz, bazz)")
}

func TestResourceSqlTableCreateStatement_Tags(t *testing.T) {
ti := &SqlTableInfo{
Name: "bar",
CatalogName: "main",
SchemaName: "foo",
TableType: "TABLE",
DataSourceFormat: "DELTA",
StorageLocation: "s3://ext-main/foo/bar1",
StorageCredentialName: "somecred",
Comment: "terraform managed",
Tags: map[string]string{"foo": "bar", "fizz": "buzz"},
}
stmt := ti.buildTableCreateStatement()
assert.Contains(t, stmt, "CREATE TABLE `main`.`foo`.`bar`")
assert.Contains(t, stmt, "USING DELTA")
assert.Contains(t, stmt, "LOCATION 's3://ext-main/foo/bar1' WITH (CREDENTIAL `somecred`)")
assert.Contains(t, stmt, "COMMENT 'terraform managed'")
assert.Contains(t, stmt, ";ALTER TABLE `main`.`foo`.`bar` SET TAGS (")
assert.Contains(t, stmt, "'foo'='bar'")
assert.Contains(t, stmt, "'fizz'='buzz'")
}

func TestResourceSqlTableCreateStatement_Liquid(t *testing.T) {
ti := &SqlTableInfo{
Name: "bar",
Expand Down Expand Up @@ -209,6 +231,10 @@ func TestResourceSqlTableCreateTable(t *testing.T) {
comment = "name of thing"
}
comment = "this table is managed by terraform"
tags = {
fizz = "buzz"
foo = "bar"
}
`,
Fixtures: append([]qa.HTTPFixture{
{
Expand All @@ -227,6 +253,10 @@ func TestResourceSqlTableCreateTable(t *testing.T) {
"one": "two",
"three": "four",
},
Tags: map[string]string{
"foo": "bar",
"fizz": "buzz",
},
},
},
}, useExistingClusterForSql...),
Expand Down Expand Up @@ -1082,6 +1112,17 @@ func TestResourceSqlTableUpdateTable_ColumnsTypeUpperLowerCaseThrowsError(t *tes
)
}

func TestResourceSqlTableSerializeTags(t *testing.T) {
ti := &SqlTableInfo{
Tags: map[string]string{
"one": "two",
"three": "four",
},
}
assert.Contains(t, ti.serializeTags(), "'one'='two'")
assert.Contains(t, ti.serializeTags(), "'three'='four'")
}

func TestResourceSqlTableUpdateTable_ColumnsAdditionAndUpdateThrowsError(t *testing.T) {
resourceSqlTableUpdateColumnHelper(t,
resourceSqlTableUpdateColumnTestMetaData{
Expand Down
1 change: 1 addition & 0 deletions docs/resources/sql_table.md
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ The following arguments are supported:
* `comment` - (Optional) User-supplied free-form text. Changing comment is not currently supported on `VIEW` table_type.
* `options` - (Optional) Map of user defined table options. Change forces creation of a new resource.
* `properties` - (Optional) Map of table properties.
* `tags` - (Optional) Map of table tags.
* `partitions` - (Optional) a subset of columns to partition the table by. Change forces creation of a new resource. Conflicts with `cluster_keys`. Change forces creation of a new resource.

### `column` configuration block
Expand Down
69 changes: 69 additions & 0 deletions internal/acceptance/sql_table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,75 @@
}`,
})
}
func TestUcAccResourceSqlTable_CreateTags(t *testing.T) {
if os.Getenv("GOOGLE_CREDENTIALS") != "" {
skipf(t)("databricks_sql_table resource not available on GCP")

Check failure on line 76 in internal/acceptance/sql_table_test.go

View workflow job for this annotation

GitHub Actions / tests

undefined: skipf (compile)
}
UnityWorkspaceLevel(t, Step{
Template: `
resource "databricks_schema" "this" {
name = "{var.STICKY_RANDOM}"
catalog_name = "main"
}

resource "databricks_sql_table" "this" {
name = "bar"
catalog_name = "main"
schema_name = databricks_schema.this.name
table_type = "MANAGED"
properties = {
this = "that"
something = "else"
}

column {
name = "id"
type = "int"
}
column {
name = "name"
type = "string"
}
comment = "this table is managed by terraform"
owner = "account users"
tags = {
mytag = "value"
anothertag = "anothervalue"
}
}`,
}, Step{
Template: `
resource "databricks_schema" "this" {
name = "{var.STICKY_RANDOM}"
catalog_name = "main"
}

resource "databricks_sql_table" "this" {
name = "bar"
catalog_name = "main"
schema_name = databricks_schema.this.name
table_type = "MANAGED"
properties = {
that = "this"
something = "else2"
}

column {
name = "id"
type = "int"
}
column {
name = "name"
type = "string"
}
comment = "this table is managed by terraform..."
tags = {
mytag = "value"
anothertag = "anothervalue"
}
}`,
})
}

func TestUcAccResourceSqlTableWithIdentityColumn_Managed(t *testing.T) {
if os.Getenv("GOOGLE_CREDENTIALS") != "" {
Expand Down
Loading