OB-37029: Allow edit-forward of Terraform-managed datasets

Expose saveDataset's rematerialization_mode to resource_dataset in Terraform. Must wait until 10/30 prod release for this option to work properly.

client/api.go

@@ -33,7 +33,7 @@ func (c *Client) GetDataset(ctx context.Context, id string) (*meta.Dataset, erro
 	return c.Meta.GetDataset(ctx, id)
 }
 
-func (c *Client) SaveDataset(ctx context.Context, wsid string, input *meta.DatasetInput, queryInput *meta.MultiStageQueryInput) (*meta.Dataset, error) {
+func (c *Client) SaveDataset(ctx context.Context, wsid string, input *meta.DatasetInput, queryInput *meta.MultiStageQueryInput, dependencyHandling *meta.DependencyHandlingInput) (*meta.Dataset, error) {
 	if !c.Flags[flagObs2110] {
 		c.obs2110.Lock()
 		defer c.obs2110.Unlock()
@@ -46,7 +46,7 @@ func (c *Client) SaveDataset(ctx context.Context, wsid string, input *meta.Datas
 		input.ManagedById = c.Config.ManagingObjectID
 	}
 
-	return c.Meta.SaveDataset(ctx, wsid, input, queryInput)
+	return c.Meta.SaveDataset(ctx, wsid, input, queryInput, dependencyHandling)
 }
 
 // DeleteDataset by ID

client/internal/meta/schema/dataset.graphql

@@ -248,12 +248,32 @@ enum SaveMode @goModel(model: "observe/meta/metatypes.SaveMode") {
     PreflightDatasetAndDependencies
 }
 
+"""
+Specifies what type of rematerialization will occur when a dataset is updated
+"""
+enum RematerializationMode @goModel(model: "observe/meta/metatypes.RematerializationMode") {
+    """
+    Rematerialize dataset and all downstream dependencies
+    """
+    Rematerialize
+    """
+    Skips rematerialization if certain conditions are met, will rematerialize otherwise. Use with
+    SaveMode.PreflightDataset to verify rematerialization will not occur for a given dataset update
+    before updating the dataset.
+    """
+    SkipRematerialization
+}
+
 input DependencyHandlingInput @goModel(model: "observe/meta/metatypes.DependencyHandling") {
     saveMode: SaveMode
     """
     For saveMode UpdateDatasetAndDependenciesUnlessNewErrors, here are errors that don't count as "new"
     """
     ignoreSpecificErrors: [ObjectId!]
+    """
+    If no mode is specified, Rematerialize will be used by default
+    """
+    rematerializationMode: RematerializationMode
 }
 
 type ForeignKey @goModel(model: "observe/meta/metatypes.ForeignKey") {

client/meta/dataset.go

@@ -25,14 +25,20 @@ func datasetOrError(d datasetResponse, err error) (*Dataset, error) {
 	return d.GetDataset(), nil
 }
 
-func dep() *DependencyHandlingInput {
+func DefaultDependencyHandling() *DependencyHandlingInput {
 	mode := SaveModeUpdateDatasetAndDependenciesIgnoringAllErrors
 	return &DependencyHandlingInput{SaveMode: &mode}
 }
 
+func DependencyHandlingSkipRematerialization() *DependencyHandlingInput {
+	saveMode := SaveModeUpdateDatasetAndDependenciesIgnoringAllErrors
+	rematMode := RematerializationModeSkiprematerialization
+	return &DependencyHandlingInput{SaveMode: &saveMode, RematerializationMode: &rematMode}
+}
+
 // SaveDataset creates and updates datasets
-func (client *Client) SaveDataset(ctx context.Context, workspaceId string, input *DatasetInput, queryInput *MultiStageQueryInput) (*Dataset, error) {
-	resp, err := saveDataset(ctx, client.Gql, workspaceId, *input, *queryInput, dep())
+func (client *Client) SaveDataset(ctx context.Context, workspaceId string, input *DatasetInput, queryInput *MultiStageQueryInput, dependencyHandling *DependencyHandlingInput) (*Dataset, error) {
+	resp, err := saveDataset(ctx, client.Gql, workspaceId, *input, *queryInput, dependencyHandling)
 	return datasetOrError(resp.Dataset, err)
 }
 
@@ -44,7 +50,7 @@ func (client *Client) GetDataset(ctx context.Context, id string) (*Dataset, erro
 
 // DeleteDataset deletes dataset by ID.
 func (client *Client) DeleteDataset(ctx context.Context, id string) error {
-	resp, err := deleteDataset(ctx, client.Gql, id, dep())
+	resp, err := deleteDataset(ctx, client.Gql, id, DefaultDependencyHandling())
 	return optionalResultStatusError(resp, err)
 }
 
@@ -83,7 +89,7 @@ func (client *Client) ListDatasetsIdNameOnly(ctx context.Context) (ds []*Dataset
 }
 
 func (client *Client) SaveSourceDataset(ctx context.Context, workspaceId string, input *DatasetDefinitionInput, sourceInput *SourceTableDefinitionInput) (*Dataset, error) {
-	resp, err := saveSourceDataset(ctx, client.Gql, workspaceId, *input, *sourceInput, dep())
+	resp, err := saveSourceDataset(ctx, client.Gql, workspaceId, *input, *sourceInput, DefaultDependencyHandling())
 	return datasetOrError(resp.Dataset, err)
 }
 

docs/resources/dataset.md

@@ -59,6 +59,9 @@ frequency with which queries are run, which incurs higher transform costs.
 - `path_cost` (Number) Path cost incurred by this dataset when computing graph link. Increasing
 this value will reduce the preference for using this dataset when computing
 paths between two datasets.
+- `rematerialization_mode` (String) Specifies rematerialization mode when updating a dataset. Options include
+"rematerialize" and "skip_rematerialization". If no option is used, "rematerialize"
+is used by default.
 
 ### Read-Only
 

examples/data-sources/observe_dataset/data-source.tf

@@ -2,7 +2,7 @@ data "observe_workspace" "default" {
   name = "Default"
 }
 
-data "observe_datastet" "example" {
+data "observe_dataset" "example" {
   workspace = data.observe_workspace.default.oid
   name      = "My Dataset"
 }

observe/descriptions/dataset.yaml

@@ -18,3 +18,7 @@ schema:
   correlation_tag:
     description: |
       Correlation tags associated with this dataset.
+  rematerialization_mode: |
+    Specifies rematerialization mode when updating a dataset. Options include
+    "rematerialize" and "skip_rematerialization". If no option is used, "rematerialize"
+    is used by default.

observe/resource_dataset.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
+	"github.com/hashicorp/go-cty/cty"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 
@@ -22,6 +24,9 @@ const (
 	schemaDatasetDescriptionDescription = "Dataset description."
 	schemaDatasetIconDescription        = "Icon image."
 	schemaDatasetOIDDescription         = "The Observe ID for dataset."
+
+	rematerializationModeRematerialize         = "rematerialize"
+	rematerializationModeSkipRematerialization = "skip_rematerialization"
 )
 
 func resourceDataset() *schema.Resource {
@@ -143,6 +148,13 @@ func resourceDataset() *schema.Resource {
 					},
 				},
 			},
+			"rematerialization_mode": {
+				Type:             schema.TypeString,
+				Optional:         true,
+				Default:          "rematerialize",
+				ValidateDiagFunc: validateRematerializationMode,
+				Description:      descriptions.Get("dataset", "schema", "rematerialization_mode"),
+			},
 		},
 	}
 }
@@ -340,8 +352,20 @@ func resourceDatasetCreate(ctx context.Context, data *schema.ResourceData, meta
 		return diags
 	}
 
+	dependencyHandling := gql.DefaultDependencyHandling()
+	switch data.Get("rematerialization_mode").(string) {
+	case rematerializationModeRematerialize:
+	case rematerializationModeSkipRematerialization:
+		dependencyHandling = gql.DependencyHandlingSkipRematerialization()
+
+		diags = append(diags, diag.Diagnostic{
+			Severity: diag.Warning,
+			Summary:  "Skipping rematerialization on a new dataset is a no-op",
+		})
+	}
+
 	wsid, _ := oid.NewOID(data.Get("workspace").(string))
-	result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput)
+	result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput, dependencyHandling)
 	if err != nil {
 		diags = append(diags, diag.Diagnostic{
 			Severity: diag.Error,
@@ -384,7 +408,14 @@ func resourceDatasetUpdate(ctx context.Context, data *schema.ResourceData, meta
 	input.Id = &id
 	wsid, _ := oid.NewOID(data.Get("workspace").(string))
 
-	result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput)
+	dependencyHandling := gql.DefaultDependencyHandling()
+	switch data.Get("rematerialization_mode").(string) {
+	case rematerializationModeRematerialize:
+	case rematerializationModeSkipRematerialization:
+		dependencyHandling = gql.DependencyHandlingSkipRematerialization()
+	}
+
+	result, err := client.SaveDataset(ctx, wsid.Id, input, queryInput, dependencyHandling)
 	if err != nil {
 		diags = append(diags, diag.Diagnostic{
 			Severity: diag.Error,
@@ -429,3 +460,19 @@ func diffSuppressVersion(k, old, new string, d *schema.ResourceData) bool {
 	// ignore version
 	return oldOID.Type == newOID.Type && oldOID.Id == newOID.Id
 }
+
+func validateRematerializationMode(i interface{}, path cty.Path) diag.Diagnostics {
+	s := i.(string)
+
+	rematModes := []string{rematerializationModeRematerialize, rematerializationModeSkipRematerialization}
+	for _, rematMode := range rematModes {
+		if s == rematMode {
+			return nil
+		}
+	}
+
+	return diag.Diagnostics{diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  fmt.Sprintf("Unexpected rematerialization mode, expected one of: %s, got: %s", strings.Join(rematModes, ","), s),
+	}}
+}

observe/resource_dataset_test.go

@@ -126,6 +126,7 @@ func TestAccObserveDatasetUpdate(t *testing.T) {
 
 					acceleration_disabled = true
 					data_table_view_state = jsonencode({viewType = "Auto"})
+					rematerialization_mode = "skip_rematerialization"
 
 					stage {
 					  pipeline = <<-EOF
@@ -143,6 +144,7 @@ func TestAccObserveDatasetUpdate(t *testing.T) {
 					resource.TestCheckResourceAttr("observe_dataset.first", "stage.0.input", ""),
 					resource.TestCheckResourceAttr("observe_dataset.first", "acceleration_disabled", "true"),
 					resource.TestCheckResourceAttr("observe_dataset.first", "data_table_view_state", "{\"viewType\":\"Auto\"}"),
+					resource.TestCheckResourceAttr("observe_dataset.first", "rematerialization_mode", "skip_rematerialization"),
 				),
 			},
 			{