diff --git a/client/api.go b/client/api.go index 885bff01..63e6b8fc 100644 --- a/client/api.go +++ b/client/api.go @@ -33,7 +33,7 @@ func (c *Client) GetDataset(ctx context.Context, id string) (*meta.Dataset, erro return c.Meta.GetDataset(ctx, id) } -func (c *Client) SaveDataset(ctx context.Context, wsid string, input *meta.DatasetInput, queryInput *meta.MultiStageQueryInput) (*meta.Dataset, error) { +func (c *Client) SaveDataset(ctx context.Context, wsid string, input *meta.DatasetInput, queryInput *meta.MultiStageQueryInput, dependencyHandling *meta.DependencyHandlingInput) (*meta.Dataset, error) { if !c.Flags[flagObs2110] { c.obs2110.Lock() defer c.obs2110.Unlock() @@ -46,7 +46,7 @@ func (c *Client) SaveDataset(ctx context.Context, wsid string, input *meta.Datas input.ManagedById = c.Config.ManagingObjectID } - return c.Meta.SaveDataset(ctx, wsid, input, queryInput) + return c.Meta.SaveDataset(ctx, wsid, input, queryInput, dependencyHandling) } // DeleteDataset by ID diff --git a/client/internal/meta/schema/dataset.graphql b/client/internal/meta/schema/dataset.graphql index 049c8ed2..99362ef5 100644 --- a/client/internal/meta/schema/dataset.graphql +++ b/client/internal/meta/schema/dataset.graphql @@ -248,12 +248,32 @@ enum SaveMode @goModel(model: "observe/meta/metatypes.SaveMode") { PreflightDatasetAndDependencies } +""" +Specifies what type of rematerialization will occur when a dataset is updated +""" +enum RematerializationMode @goModel(model: "observe/meta/metatypes.RematerializationMode") { + """ + Rematerialize dataset and all downstream dependencies + """ + Rematerialize + """ + Skips rematerialization if certain conditions are met, will rematerialize otherwise. Use with + SaveMode.PreflightDataset to verify rematerialization will not occur for a given dataset update + before updating the dataset. + """ + SkipRematerialization +} + input DependencyHandlingInput @goModel(model: "observe/meta/metatypes.DependencyHandling") { saveMode: SaveMode """ For saveMode UpdateDatasetAndDependenciesUnlessNewErrors, here are errors that don't count as "new" """ ignoreSpecificErrors: [ObjectId!] + """ + If no mode is specified, Rematerialize will be used by default + """ + rematerializationMode: RematerializationMode } type ForeignKey @goModel(model: "observe/meta/metatypes.ForeignKey") { diff --git a/client/meta/dataset.go b/client/meta/dataset.go index d84bff9c..add2ac4d 100644 --- a/client/meta/dataset.go +++ b/client/meta/dataset.go @@ -25,14 +25,20 @@ func datasetOrError(d datasetResponse, err error) (*Dataset, error) { return d.GetDataset(), nil } -func dep() *DependencyHandlingInput { +func DefaultDependencyHandling() *DependencyHandlingInput { mode := SaveModeUpdateDatasetAndDependenciesIgnoringAllErrors return &DependencyHandlingInput{SaveMode: &mode} } +func DependencyHandlingSkipRematerialization() *DependencyHandlingInput { + saveMode := SaveModeUpdateDatasetAndDependenciesIgnoringAllErrors + rematMode := RematerializationModeSkiprematerialization + return &DependencyHandlingInput{SaveMode: &saveMode, RematerializationMode: &rematMode} +} + // SaveDataset creates and updates datasets -func (client *Client) SaveDataset(ctx context.Context, workspaceId string, input *DatasetInput, queryInput *MultiStageQueryInput) (*Dataset, error) { - resp, err := saveDataset(ctx, client.Gql, workspaceId, *input, *queryInput, dep()) +func (client *Client) SaveDataset(ctx context.Context, workspaceId string, input *DatasetInput, queryInput *MultiStageQueryInput, dependencyHandling *DependencyHandlingInput) (*Dataset, error) { + resp, err := saveDataset(ctx, client.Gql, workspaceId, *input, *queryInput, dependencyHandling) return datasetOrError(resp.Dataset, err) } @@ -44,7 +50,7 @@ func (client *Client) GetDataset(ctx context.Context, id string) (*Dataset, erro // DeleteDataset deletes dataset by ID. func (client *Client) DeleteDataset(ctx context.Context, id string) error { - resp, err := deleteDataset(ctx, client.Gql, id, dep()) + resp, err := deleteDataset(ctx, client.Gql, id, DefaultDependencyHandling()) return optionalResultStatusError(resp, err) } @@ -83,7 +89,7 @@ func (client *Client) ListDatasetsIdNameOnly(ctx context.Context) (ds []*Dataset } func (client *Client) SaveSourceDataset(ctx context.Context, workspaceId string, input *DatasetDefinitionInput, sourceInput *SourceTableDefinitionInput) (*Dataset, error) { - resp, err := saveSourceDataset(ctx, client.Gql, workspaceId, *input, *sourceInput, dep()) + resp, err := saveSourceDataset(ctx, client.Gql, workspaceId, *input, *sourceInput, DefaultDependencyHandling()) return datasetOrError(resp.Dataset, err) } diff --git a/client/meta/genqlient.generated.go b/client/meta/genqlient.generated.go index c8f6162f..9a52f3ba 100644 --- a/client/meta/genqlient.generated.go +++ b/client/meta/genqlient.generated.go @@ -2205,6 +2205,8 @@ type DependencyHandlingInput struct { SaveMode *SaveMode `json:"saveMode"` // For saveMode UpdateDatasetAndDependenciesUnlessNewErrors, here are errors that don't count as "new" IgnoreSpecificErrors []string `json:"ignoreSpecificErrors"` + // If no mode is specified, Rematerialize will be used by default + RematerializationMode *RematerializationMode `json:"rematerializationMode"` } // GetSaveMode returns DependencyHandlingInput.SaveMode, and is useful for accessing the field via an interface. @@ -2213,6 +2215,11 @@ func (v *DependencyHandlingInput) GetSaveMode() *SaveMode { return v.SaveMode } // GetIgnoreSpecificErrors returns DependencyHandlingInput.IgnoreSpecificErrors, and is useful for accessing the field via an interface. func (v *DependencyHandlingInput) GetIgnoreSpecificErrors() []string { return v.IgnoreSpecificErrors } +// GetRematerializationMode returns DependencyHandlingInput.RematerializationMode, and is useful for accessing the field via an interface. +func (v *DependencyHandlingInput) GetRematerializationMode() *RematerializationMode { + return v.RematerializationMode +} + type EmailActionInput struct { TargetUsers []types.UserIdScalar `json:"targetUsers"` TargetAddresses []string `json:"targetAddresses"` @@ -7370,6 +7377,18 @@ func (v *RbacSubjectInput) GetGroupId() *string { return v.GroupId } // GetAll returns RbacSubjectInput.All, and is useful for accessing the field via an interface. func (v *RbacSubjectInput) GetAll() *bool { return v.All } +// Specifies what type of rematerialization will occur when a dataset is updated +type RematerializationMode string + +const ( + // Rematerialize dataset and all downstream dependencies + RematerializationModeRematerialize RematerializationMode = "Rematerialize" + // Skips rematerialization if certain conditions are met, will rematerialize otherwise. Use with + // SaveMode.PreflightDataset to verify rematerialization will not occur for a given dataset update + // before updating the dataset. + RematerializationModeSkiprematerialization RematerializationMode = "SkipRematerialization" +) + type ResourceIdInput struct { DatasetId string `json:"datasetId"` PrimaryKeyValue []ColumnAndValueInput `json:"primaryKeyValue"` diff --git a/docs/resources/dataset.md b/docs/resources/dataset.md index 72d5ce17..b7225412 100644 --- a/docs/resources/dataset.md +++ b/docs/resources/dataset.md @@ -59,6 +59,9 @@ frequency with which queries are run, which incurs higher transform costs. - `path_cost` (Number) Path cost incurred by this dataset when computing graph link. Increasing this value will reduce the preference for using this dataset when computing paths between two datasets. +- `rematerialization_mode` (String) Specifies rematerialization mode when updating a dataset. Options include +"rematerialize" and "skip_rematerialization". If no option is used, "rematerialize" +is used by default. ### Read-Only diff --git a/examples/data-sources/observe_dataset/data-source.tf b/examples/data-sources/observe_dataset/data-source.tf index ee464345..1c0df7bf 100644 --- a/examples/data-sources/observe_dataset/data-source.tf +++ b/examples/data-sources/observe_dataset/data-source.tf @@ -2,7 +2,7 @@ data "observe_workspace" "default" { name = "Default" } -data "observe_datastet" "example" { +data "observe_dataset" "example" { workspace = data.observe_workspace.default.oid name = "My Dataset" } diff --git a/observe/descriptions/dataset.yaml b/observe/descriptions/dataset.yaml index a5797d11..13663490 100644 --- a/observe/descriptions/dataset.yaml +++ b/observe/descriptions/dataset.yaml @@ -18,3 +18,7 @@ schema: correlation_tag: description: | Correlation tags associated with this dataset. + rematerialization_mode: | + Specifies rematerialization mode when updating a dataset. Options include + "rematerialize" and "skip_rematerialization". If no option is used, "rematerialize" + is used by default. diff --git a/observe/resource_dataset.go b/observe/resource_dataset.go index edff8753..d09babfc 100644 --- a/observe/resource_dataset.go +++ b/observe/resource_dataset.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "log" + "strings" "time" + "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -22,6 +24,9 @@ const ( schemaDatasetDescriptionDescription = "Dataset description." schemaDatasetIconDescription = "Icon image." schemaDatasetOIDDescription = "The Observe ID for dataset." + + rematerializationModeRematerialize = "rematerialize" + rematerializationModeSkipRematerialization = "skip_rematerialization" ) func resourceDataset() *schema.Resource { @@ -143,6 +148,13 @@ func resourceDataset() *schema.Resource { }, }, }, + "rematerialization_mode": { + Type: schema.TypeString, + Optional: true, + Default: "rematerialize", + ValidateDiagFunc: validateRematerializationMode, + Description: descriptions.Get("dataset", "schema", "rematerialization_mode"), + }, }, } } @@ -340,8 +352,20 @@ func resourceDatasetCreate(ctx context.Context, data *schema.ResourceData, meta return diags } + dependencyHandling := gql.DefaultDependencyHandling() + switch data.Get("rematerialization_mode").(string) { + case rematerializationModeRematerialize: + case rematerializationModeSkipRematerialization: + dependencyHandling = gql.DependencyHandlingSkipRematerialization() + + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "Skipping rematerialization on a new dataset is a no-op", + }) + } + wsid, _ := oid.NewOID(data.Get("workspace").(string)) - result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput) + result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput, dependencyHandling) if err != nil { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, @@ -384,7 +408,14 @@ func resourceDatasetUpdate(ctx context.Context, data *schema.ResourceData, meta input.Id = &id wsid, _ := oid.NewOID(data.Get("workspace").(string)) - result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput) + dependencyHandling := gql.DefaultDependencyHandling() + switch data.Get("rematerialization_mode").(string) { + case rematerializationModeRematerialize: + case rematerializationModeSkipRematerialization: + dependencyHandling = gql.DependencyHandlingSkipRematerialization() + } + + result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput, dependencyHandling) if err != nil { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, @@ -429,3 +460,19 @@ func diffSuppressVersion(k, old, new string, d *schema.ResourceData) bool { // ignore version return oldOID.Type == newOID.Type && oldOID.Id == newOID.Id } + +func validateRematerializationMode(i interface{}, path cty.Path) diag.Diagnostics { + s := i.(string) + + rematModes := []string{rematerializationModeRematerialize, rematerializationModeSkipRematerialization} + for _, rematMode := range rematModes { + if s == rematMode { + return nil + } + } + + return diag.Diagnostics{diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("Unexpected rematerialization mode, expected one of: %s, got: %s", strings.Join(rematModes, ","), s), + }} +} diff --git a/observe/resource_dataset_test.go b/observe/resource_dataset_test.go index be2f8db0..feb089a3 100644 --- a/observe/resource_dataset_test.go +++ b/observe/resource_dataset_test.go @@ -126,6 +126,7 @@ func TestAccObserveDatasetUpdate(t *testing.T) { acceleration_disabled = true data_table_view_state = jsonencode({viewType = "Auto"}) + rematerialization_mode = "skip_rematerialization" stage { pipeline = <<-EOF @@ -143,6 +144,7 @@ func TestAccObserveDatasetUpdate(t *testing.T) { resource.TestCheckResourceAttr("observe_dataset.first", "stage.0.input", ""), resource.TestCheckResourceAttr("observe_dataset.first", "acceleration_disabled", "true"), resource.TestCheckResourceAttr("observe_dataset.first", "data_table_view_state", "{\"viewType\":\"Auto\"}"), + resource.TestCheckResourceAttr("observe_dataset.first", "rematerialization_mode", "skip_rematerialization"), ), }, {