Add binding export functionality to dashboards

This is an internal use options that allows id references to be
collected in a unified location in the resulting datasource read schema
for further postprocessing.

client/api.go

@@ -73,6 +73,12 @@ func (c *Client) CreateSourceDataset(ctx context.Context, workspaceId string, da
 	return c.Meta.SaveSourceDataset(ctx, workspaceId, dataset, table)
 }
 
+// List all datasets, but only asks for id and name to prevent looping in expensive
+// resolvers
+func (c *Client) ListDatasetsIdNameOnly(ctx context.Context) ([]*meta.DatasetIdName, error) {
+	return c.Meta.ListDatasetsIdNameOnly(ctx)
+}
+
 // UpdateSourceDataset updates the existing source dataset
 func (c *Client) UpdateSourceDataset(ctx context.Context, workspaceId string, id string, dataset *meta.DatasetDefinitionInput, table *meta.SourceTableDefinitionInput) (*meta.Dataset, error) {
 	if !c.Flags[flagObs2110] {
@@ -811,6 +817,12 @@ func (c *Client) GetWorksheet(ctx context.Context, id string) (*meta.Worksheet,
 	return c.Meta.GetWorksheet(ctx, id)
 }
 
+// List all worksheets, but only fetch ids and labels to prevent using expensive
+// resolvers
+func (c *Client) ListWorksheetIdLabelOnly(ctx context.Context, workspaceId string) ([]*meta.WorksheetIdLabel, error) {
+	return c.Meta.ListWorksheetIdLabelOnly(ctx, workspaceId)
+}
+
 // UpdateWorksheet updates a worksheet
 // XXX: this should not have to take workspaceId, but API forces us to
 func (c *Client) UpdateWorksheet(ctx context.Context, id string, workspaceId string, input *meta.WorksheetInput) (*meta.Worksheet, error) {
@@ -1215,6 +1227,11 @@ func (c *Client) LookupUser(ctx context.Context, email string) (*meta.User, erro
 	return c.Meta.LookupUser(ctx, email)
 }
 
+// List all users in current customer
+func (c *Client) ListUsers(ctx context.Context) ([]meta.User, error) {
+	return c.Meta.ListUsers(ctx)
+}
+
 // CreateRbacGroupmember creates an rbacgroupmember
 func (c *Client) CreateRbacGroupmember(ctx context.Context, input *meta.RbacGroupmemberInput) (*meta.RbacGroupmember, error) {
 	if !c.Flags[flagObs2110] {

client/binding/binding.go

@@ -0,0 +1,248 @@
+package binding
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+
+	observe "github.com/observeinc/terraform-provider-observe/client"
+	"github.com/observeinc/terraform-provider-observe/client/meta/types"
+)
+
+var invalidCharsRegex = regexp.MustCompile(`[^A-Za-z0-9_-]`)
+var startsWithNumRegex = regexp.MustCompile(`^[0-9]`)
+
+type ResourceCache struct {
+	idToLabel Mapping
+}
+
+type NewResourceCacheOptArgs struct {
+	workspaceId string
+}
+
+func NewResourceCache(ctx context.Context, kinds KindSet, client *observe.Client, optArgs NewResourceCacheOptArgs) (ResourceCache, error) {
+	var cache = ResourceCache{idToLabel: make(Mapping)}
+	for resourceKind := range kinds {
+		resourceNames := make(map[string]struct{})
+		disambiguator := 0
+		switch resourceKind {
+		case KindDataset:
+			datasets, err := client.ListDatasetsIdNameOnly(ctx)
+			if err != nil {
+				return cache, err
+			}
+			for _, ds := range datasets {
+				cache.addEntry(KindDataset, ds.Name, ds.Id, &disambiguator, resourceNames)
+			}
+		case KindWorksheet:
+			worksheets, err := client.ListWorksheetIdLabelOnly(ctx, optArgs.workspaceId)
+			if err != nil {
+				return cache, err
+			}
+			for _, wk := range worksheets {
+				cache.addEntry(KindWorksheet, wk.Label, wk.Id, &disambiguator, resourceNames)
+			}
+		case KindWorkspace:
+			workspaces, err := client.ListWorkspaces(ctx)
+			if err != nil {
+				return cache, err
+			}
+			for _, workspace := range workspaces {
+				cache.addEntry(KindWorkspace, workspace.Label, workspace.Id, &disambiguator, resourceNames)
+			}
+		case KindUser:
+			users, err := client.ListUsers(ctx)
+			if err != nil {
+				return cache, err
+			}
+			for _, user := range users {
+				cache.addEntry(KindUser, user.Label, user.Id.String(), &disambiguator, resourceNames)
+			}
+		}
+	}
+	return cache, nil
+}
+
+func (c *ResourceCache) addEntry(kind Kind, label string, id string, disambiguator *int, existingNames map[string]struct{}) {
+	resourceName := invalidCharsRegex.ReplaceAllString(label, "_")
+	if startsWithNumRegex.FindString(resourceName) != "" {
+		resourceName = "_" + resourceName
+	}
+	if _, found := existingNames[resourceName]; found {
+		resourceName = fmt.Sprintf("%s_%d", resourceName, *disambiguator)
+		*disambiguator++
+	}
+	c.idToLabel[Ref{kind: kind, key: id}] = Target{
+		TfName: resourceName,
+		Value:  label,
+	}
+}
+
+func (c *ResourceCache) LookupId(kind Kind, id string) *Target {
+	maybeLabel, ok := c.idToLabel[Ref{kind: kind, key: id}]
+	if !ok {
+		return nil
+	}
+	return &maybeLabel
+}
+
+type Generator struct {
+	enabled         bool
+	resourceType    string
+	resourceName    string
+	enabledBindings KindSet
+	client          *observe.Client
+	bindings        Mapping
+	cache           ResourceCache
+}
+
+func NewGenerator(ctx context.Context, enabled bool, resourceType string, resourceName string,
+	client *observe.Client, enabledBindings KindSet) (Generator, error) {
+	enabled = enabled && client.Config.ExportObjectBindings
+	if !enabled {
+		return Generator{enabled: false}, nil
+	}
+	rc, err := NewResourceCache(ctx, enabledBindings, client, NewResourceCacheOptArgs{})
+	if err != nil {
+		return Generator{}, err
+	}
+	bindings := NewMapping()
+	return Generator{
+		enabled:         true,
+		resourceType:    resourceType,
+		resourceName:    resourceName,
+		enabledBindings: enabledBindings,
+		client:          client,
+		bindings:        bindings,
+		cache:           rc,
+	}, nil
+}
+
+func (g *Generator) Generate(ctx context.Context, data interface{}) {
+	mapOverJsonStringKeys(data, func(key string, value string, jsonMapNode map[string]interface{}) {
+		kinds := resolveKeyToKinds(key)
+		for _, kind := range kinds {
+			// if not enabled, skip
+			if _, found := g.enabledBindings[kind]; !found {
+				continue
+			}
+			// Lookup resource in cache
+			maybeLabel := g.cache.LookupId(kind, value)
+			// update with binding
+			if maybeLabel != nil {
+				b := Ref{kind: kind, key: maybeLabel.Value}
+				terraformLocal := fmt.Sprintf("binding__%s_%s__%s_%s", g.resourceType, g.resourceName, kind, maybeLabel.TfName)
+				g.bindings[b] = Target{
+					Value:             value,
+					TfName:            maybeLabel.TfName,
+					TfLocalBindingVar: terraformLocal,
+				}
+				jsonMapNode[key] = fmt.Sprintf("${local.%s}", terraformLocal)
+				break
+			}
+		}
+	})
+}
+
+func (g *Generator) GenerateJson(ctx context.Context, jsonStr []byte) ([]byte, error) {
+	if !g.enabled {
+		return jsonStr, nil
+	}
+	serialized, err := transformJson(jsonStr, func(dataPtr *interface{}) error {
+		g.Generate(ctx, *dataPtr)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return serialized, nil
+}
+
+func (g *Generator) InsertBindingsObject(data map[string]interface{}) {
+	enabledList := make([]Kind, 0)
+	for binding := range g.enabledBindings {
+		enabledList = append(enabledList, binding)
+	}
+	bindingsObject := BindingsObject{
+		Name:     g.resourceName,
+		Mappings: g.bindings,
+		Kinds:    enabledList,
+	}
+	data[bindingsKey] = bindingsObject
+}
+
+func (g *Generator) InsertBindingsObjectJson(jsonData *types.JsonObject) (*types.JsonObject, error) {
+	if !g.enabled {
+		return jsonData, nil
+	}
+	serialized, err := transformJson([]byte(jsonData.String()), func(dataPtr *interface{}) error {
+		g.InsertBindingsObject((*dataPtr).(map[string]interface{}))
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return types.JsonObject(serialized).Ptr(), nil
+}
+
+func resolveKeyToKinds(key string) []Kind {
+	switch key {
+	case "id":
+		return []Kind{KindDataset, KindWorksheet}
+	case "datasetId":
+		fallthrough
+	case "targetDataset":
+		fallthrough
+	case "dataset":
+		return []Kind{KindDataset}
+	default:
+		return []Kind{}
+	}
+}
+
+func mapOverJsonStringKeys(data interface{}, f func(key string, value string, jsonMapNode map[string]interface{})) {
+	var stack []interface{}
+	stack = append(stack, data)
+	for len(stack) > 0 {
+		var cur interface{}
+		cur, stack = stack[len(stack)-1], stack[:len(stack)-1]
+		switch jsonNode := cur.(type) {
+		case map[string]interface{}:
+			for k, v := range jsonNode {
+				switch kvValue := v.(type) {
+				case string:
+					f(k, kvValue, jsonNode)
+				// if value looks like a composite type, push onto stack for further
+				// processing
+				case map[string]interface{}:
+					stack = append(stack, kvValue)
+				case []interface{}:
+					stack = append(stack, kvValue)
+				}
+			}
+		case []interface{}:
+			for _, object := range jsonNode {
+				stack = append(stack, object)
+			}
+		}
+	}
+}
+
+func transformJson(data []byte, f func(data *interface{}) error) ([]byte, error) {
+	var deserialized interface{}
+	err := json.Unmarshal(data, &deserialized)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to deserialize json: %w", err)
+	}
+	err = f(&deserialized)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to transform json data: %w", err)
+
+	}
+	serialized, err := json.Marshal(deserialized)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to serialize json data: %w", err)
+	}
+	return serialized, nil
+}

client/binding/types.go

@@ -0,0 +1,91 @@
+package binding
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type Ref struct {
+	kind Kind
+	key  string
+}
+
+type Target struct {
+	TfLocalBindingVar string `json:"tf_local_binding_var"`
+	TfName            string `json:"tf_name"`
+	Value             string `json:"value"`
+}
+
+type Mapping map[Ref]Target
+
+type Kind string
+
+type KindSet map[Kind]struct{}
+
+type BindingsObject struct {
+	Name     string  `json:"name"`
+	Mappings Mapping `json:"mappings"`
+	Kinds    []Kind  `json:"kinds"`
+}
+
+const (
+	KindDataset   Kind = "dataset"
+	KindWorksheet Kind = "worksheet"
+	KindWorkspace Kind = "workspace"
+	KindUser      Kind = "user"
+)
+
+const (
+	bindingsKey = "bindings"
+)
+
+var bindingRefParseRegex = regexp.MustCompile(`(.*):(.*)`)
+
+var allKinds = NewKindSet(
+	KindDataset,
+	KindWorksheet,
+	KindWorkspace,
+	KindUser,
+)
+
+func (r *Ref) String() string {
+	return fmt.Sprintf("%s:%s", r.kind, r.key)
+}
+
+func (r Ref) MarshalText() (text []byte, err error) {
+	return []byte(r.String()), nil
+}
+
+func (r *Ref) UnmarshalText(text []byte) error {
+	ref, ok := NewRefFromString(string(text))
+	if !ok {
+		return fmt.Errorf("failed to unmarshal reference type")
+	}
+	*r = ref
+	return nil
+}
+
+func NewRefFromString(s string) (Ref, bool) {
+	matches := bindingRefParseRegex.FindStringSubmatch(s)
+	if len(matches) == 0 {
+		return Ref{}, false
+	}
+	maybeKind := Kind(matches[1])
+	if _, ok := allKinds[maybeKind]; !ok {
+		return Ref{}, false
+	}
+	return Ref{kind: maybeKind, key: matches[2]}, true
+}
+
+func NewMapping() Mapping {
+	return make(Mapping)
+}
+
+func NewKindSet(kinds ...Kind) KindSet {
+	set := make(KindSet)
+	var empty struct{}
+	for _, kind := range kinds {
+		set[kind] = empty
+	}
+	return set
+}

client/config.go

@@ -52,6 +52,9 @@ type Config struct {
 
 	// optional traceparent identifier to pass via header
 	TraceParent *string `json:"traceparent"`
+
+	// enable extra queries needed to export bindings
+	ExportObjectBindings bool `json:"export_object_bindings"`
 }
 
 func (c *Config) Hash() uint64 {