Properties (#798)

pkg/construct2/properties.go

@@ -432,6 +432,23 @@ func (i *mapValuePathItem) Remove(value any) (err error) {
 		arr = arr.Elem()
 	}
 	if arr.Kind() != reflect.Slice && arr.Kind() != reflect.Array {
+		if hs, ok := arr.Interface().(set.HashedSet[string, any]); ok {
+			if hs.Contains(value) {
+				removed := hs.Remove(value)
+				if !removed {
+					return &PropertyPathError{
+						Path:  itemToPath(i),
+						Cause: fmt.Errorf("value %v not removed from set", value),
+					}
+				}
+			} else {
+				return &PropertyPathError{
+					Path:  itemToPath(i),
+					Cause: fmt.Errorf("value %v not found in set", value),
+				}
+			}
+			return nil
+		}
 		return &PropertyPathError{
 			Path:  itemToPath(i),
 			Cause: fmt.Errorf("for non-nil value'd (%v), must be array (got %s) to remove by value", value, arr.Type()),

pkg/construct2/resource_id.go

@@ -67,6 +67,15 @@ func (l *ResourceList) UnmarshalText(b []byte) error {
 	return fmt.Errorf("could not unmarshal resource list: %s", string(b))
 }
 
+func (l ResourceList) MatchesAny(id ResourceId) bool {
+	for _, rid := range l {
+		if rid.Matches(id) {
+			return true
+		}
+	}
+	return false
+}
+
 var zeroId = ResourceId{}
 
 func (id ResourceId) IsZero() bool {

pkg/engine2/cli.go

@@ -5,15 +5,18 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"reflect"
 	"runtime/pprof"
 	"strings"
 
+	"github.com/iancoleman/strcase"
 	"github.com/klothoplatform/klotho/pkg/analytics"
 	"github.com/klothoplatform/klotho/pkg/closenicely"
 	construct "github.com/klothoplatform/klotho/pkg/construct2"
 	"github.com/klothoplatform/klotho/pkg/engine2/constraints"
 	"github.com/klothoplatform/klotho/pkg/io"
 	knowledgebase "github.com/klothoplatform/klotho/pkg/knowledge_base2"
+	"github.com/klothoplatform/klotho/pkg/knowledge_base2/reader"
 	"github.com/klothoplatform/klotho/pkg/logging"
 	"github.com/klothoplatform/klotho/pkg/templates"
 	"github.com/pkg/errors"
@@ -125,7 +128,7 @@ func (em *EngineMain) AddEngineCli(root *cobra.Command) {
 }
 
 func (em *EngineMain) AddEngine() error {
-	kb, err := knowledgebase.NewKBFromFs(templates.ResourceTemplates, templates.EdgeTemplates, templates.Models)
+	kb, err := reader.NewKBFromFs(templates.ResourceTemplates, templates.EdgeTemplates, templates.Models)
 	if err != nil {
 		return err
 	}
@@ -140,17 +143,27 @@ type resourceInfo struct {
 	Views           map[string]string `json:"views"`
 }
 
-func addSubProperties(properties map[string]any, subProperties map[string]*knowledgebase.Property) {
+var validationFields = []string{"MinLength", "MaxLength", "MinValue", "MaxValue", "AllowedValues"}
+
+func addSubProperties(properties map[string]any, subProperties map[string]knowledgebase.Property) {
 	for _, subProperty := range subProperties {
-		properties[subProperty.Name] = map[string]any{
-			"type":                  subProperty.Type,
-			"deployTime":            subProperty.DeployTime,
-			"configurationDisabled": subProperty.ConfigurationDisabled,
-			"required":              subProperty.Required,
+		details := subProperty.Details()
+		properties[details.Name] = map[string]any{
+			"type":                  subProperty.Type(),
+			"deployTime":            details.DeployTime,
+			"configurationDisabled": details.ConfigurationDisabled,
+			"required":              details.Required,
+		}
+		for _, validationField := range validationFields {
+			valField := reflect.ValueOf(subProperty).Elem().FieldByName(validationField)
+			if valField.IsValid() && !valField.IsZero() {
+				val := valField.Interface()
+				properties[details.Name].(map[string]any)[strcase.ToLowerCamel(validationField)] = val
+			}
 		}
-		if subProperty.Properties != nil {
-			properties[subProperty.Name].(map[string]any)["properties"] = map[string]any{}
-			addSubProperties(properties[subProperty.Name].(map[string]any)["properties"].(map[string]any), subProperty.Properties)
+		if subProperty.SubProperties() != nil {
+			properties[details.Name].(map[string]any)["properties"] = map[string]any{}
+			addSubProperties(properties[details.Name].(map[string]any)["properties"].(map[string]any), subProperty.SubProperties())
 		}
 	}
 
@@ -166,18 +179,7 @@ func (em *EngineMain) ListResourceTypes(cmd *cobra.Command, args []string) error
 
 	for _, resourceType := range resourceTypes {
 		properties := map[string]any{}
-		for _, property := range resourceType.Properties {
-			properties[property.Name] = map[string]any{
-				"type":                  property.Type,
-				"deployTime":            property.DeployTime,
-				"configurationDisabled": property.ConfigurationDisabled,
-				"required":              property.Required,
-			}
-			if property.Properties != nil {
-				properties[property.Name].(map[string]any)["properties"] = map[string]any{}
-				addSubProperties(properties[property.Name].(map[string]any)["properties"].(map[string]any), property.Properties)
-			}
-		}
+		addSubProperties(properties, resourceType.Properties)
 		typeAndClassifications[resourceType.QualifiedTypeName] = resourceInfo{
 			Classifications: resourceType.Classification.Is,
 			Properties:      properties,

pkg/engine2/operational_eval/dependency_capture.go

@@ -165,11 +165,40 @@ func (ctx *fauxConfigContext) ExecuteOpRule(
 		exec(opRule.If)
 	}
 	for _, step := range opRule.Steps {
-		for _, stepRes := range step.Resources {
-			exec(stepRes.Selector)
-			for _, propValue := range stepRes.Properties {
-				exec(propValue)
-			}
+		errs = errors.Join(errs, ctx.executeOpStep(data, step))
+	}
+	return errs
+}
+
+func (ctx *fauxConfigContext) ExecutePropertyRule(
+	data knowledgebase.DynamicValueData,
+	propRule knowledgebase.PropertyRule,
+) error {
+	var errs error
+	exec := func(v any) {
+		errs = errors.Join(errs, ctx.Execute(v, data))
+	}
+	exec(propRule.If)
+	if propRule.Value != nil {
+		ctx.ExecuteValue(propRule.Value, data)
+	}
+	errs = errors.Join(errs, ctx.executeOpStep(data, propRule.Step))
+
+	return errs
+}
+
+func (ctx *fauxConfigContext) executeOpStep(
+	data knowledgebase.DynamicValueData,
+	step knowledgebase.OperationalStep,
+) error {
+	var errs error
+	exec := func(v any) {
+		errs = errors.Join(errs, ctx.Execute(v, data))
+	}
+	for _, stepRes := range step.Resources {
+		exec(stepRes.Selector)
+		for _, propValue := range stepRes.Properties {
+			exec(propValue)
 		}
 	}
 	return errs
@@ -251,14 +280,7 @@ func emptyValue(tmpl *knowledgebase.ResourceTemplate, property string) (any, err
 	if prop == nil {
 		return nil, fmt.Errorf("could not find property %s on template %s", property, tmpl.Id())
 	}
-	ptype, err := prop.PropertyType()
-	if err != nil {
-		return nil, fmt.Errorf(
-			"could not get property type for property %s on template %s: %w",
-			property, tmpl.Id(), err,
-		)
-	}
-	return ptype.ZeroValue(), nil
+	return prop.ZeroValue(), nil
 }
 
 func (ctx *fauxConfigContext) HasUpstream(selector any, resource construct.ResourceId) (bool, error) {

pkg/engine2/operational_eval/graph.go

@@ -144,10 +144,9 @@ func (eval *Evaluator) resourceVertices(
 ) (graphChanges, error) {
 	changes := newChanges()
 	var errs error
-
-	addProp := func(prop *knowledgebase.Property) error {
+	addProp := func(prop knowledgebase.Property) error {
 		vertex := &propertyVertex{
-			Ref:       construct.PropertyRef{Resource: res.ID, Property: prop.Path},
+			Ref:       construct.PropertyRef{Resource: res.ID, Property: prop.Details().Path},
 			Template:  prop,
 			EdgeRules: make(map[construct.SimpleEdge][]knowledgebase.OperationalRule),
 		}

pkg/engine2/operational_eval/vertex_path_expand.go

@@ -246,45 +246,29 @@ func (v *pathExpandVertex) addDepsFromProps(
 	}
 	var errs error
 	for k, prop := range tmpl.Properties {
-		pt, err := prop.PropertyType()
-		if err != nil {
-			errs = errors.Join(errs, err)
-			continue
-		}
-		if prop.OperationalRule == nil {
+		details := prop.Details()
+		if details.OperationalRule == nil {
 			// If the property can't create resources, skip it.
 			continue
 		}
-		ready, err := operational_rule.EvaluateIfCondition(*prop.OperationalRule,
+		ready, err := operational_rule.EvaluateIfCondition(details.OperationalRule.If,
 			eval.Solution, knowledgebase.DynamicValueData{Resource: res})
 		if err != nil || !ready {
 			continue
 		}
 
-		resType, ok := pt.(*knowledgebase.ResourcePropertyType)
-		if !ok {
-			listType, ok := pt.(*knowledgebase.ListPropertyType)
-			if !ok || listType.Value == "" {
+		ref := construct.PropertyRef{Resource: res, Property: k}
+		for _, dep := range dependencies {
+			if dep == v.Edge.Source || dep == v.Edge.Target {
 				continue
 			}
-			pType := knowledgebase.Property{Type: listType.Value}
-			pt, err = pType.PropertyType()
+			resource, err := eval.Solution.RawView().Vertex(res)
 			if err != nil {
 				errs = errors.Join(errs, err)
 				continue
 			}
-			resType, ok = pt.(*knowledgebase.ResourcePropertyType)
-			if !ok {
-				continue
-			}
-		}
-
-		ref := construct.PropertyRef{Resource: res, Property: k}
-		for _, dep := range dependencies {
-			if dep == v.Edge.Source || dep == v.Edge.Target {
-				continue
-			}
-			if resType.Value.Matches(dep) {
+			// if this dependency could pass validation for the resources property, consider it as a dependent vertex
+			if err := prop.Validate(resource, dep); err == nil {
 				changes.addEdge(v.Key(), Key{Ref: ref})
 			}
 		}

pkg/engine2/operational_eval/vertex_property.go

@@ -16,7 +16,7 @@ type (
 	propertyVertex struct {
 		Ref construct.PropertyRef
 
-		Template  *knowledgebase.Property
+		Template  knowledgebase.Property
 		EdgeRules map[construct.SimpleEdge][]knowledgebase.OperationalRule
 	}
 )
@@ -35,21 +35,21 @@ func (prop *propertyVertex) Dependencies(eval *Evaluator) (graphChanges, error)
 
 	// Template can be nil when checking for dependencies from a propertyVertex when adding an edge template
 	if prop.Template != nil {
-		propCtx.ExecuteValue(prop.Template.DefaultValue, resData)
-
-		if opRule := prop.Template.OperationalRule; opRule != nil {
-			if err := propCtx.ExecuteOpRule(resData, *opRule); err != nil {
+		_, _ = prop.Template.GetDefaultValue(propCtx, resData)
+		details := prop.Template.Details()
+		if opRule := details.OperationalRule; opRule != nil {
+			if err := propCtx.ExecutePropertyRule(resData, *opRule); err != nil {
 				return changes, fmt.Errorf("could not execute resource operational rule for %s: %w", prop.Ref, err)
 			}
 		}
 
-		if !prop.Template.Namespace {
+		if !details.Namespace {
 			tmpl, err := kb.GetResourceTemplate(prop.Ref.Resource)
 			if err != nil {
 				return changes, fmt.Errorf("could not get resource template for %s: %w", prop.Ref.Resource, err)
 			}
 			for propKey, propTmpl := range tmpl.Properties {
-				if propTmpl.Namespace {
+				if propTmpl.Details().Namespace {
 					nsRef := construct.PropertyRef{Resource: prop.Ref.Resource, Property: propKey}
 					propCtx.addRef(nsRef)
 				}
@@ -130,8 +130,8 @@ func (v *propertyVertex) Evaluate(eval *Evaluator) error {
 	if err := eval.UpdateId(v.Ref.Resource, res.ID); err != nil {
 		return err
 	}
-
-	if strings.HasPrefix(v.Template.Type, "list") || strings.HasPrefix(v.Template.Type, "set") {
+	propertyType := v.Template.Type()
+	if strings.HasPrefix(propertyType, "list") || strings.HasPrefix(propertyType, "set") {
 		// If we have modified a list or set we want to re add the resource to be evaluated
 		// so the nested fields are ensured to be set if required
 		return eval.AddResources(res)
@@ -160,11 +160,19 @@ func (v *propertyVertex) evaluateConstraints(sol solution_context.SolutionContex
 		return fmt.Errorf("could not get current value for %s: %w", v.Ref, err)
 	}
 
-	if currentValue == nil && setConstraint.Operator == "" && v.Template != nil && v.Template.DefaultValue != nil {
+	ctx := solution_context.DynamicCtx(sol)
+	var defaultVal any
+	if currentValue == nil {
+		defaultVal, err = v.Template.GetDefaultValue(ctx, dynData)
+		if err != nil {
+			return fmt.Errorf("could not get default value for %s: %w", v.Ref, err)
+		}
+	}
+	if currentValue == nil && setConstraint.Operator == "" && v.Template != nil && defaultVal != nil {
 		err = solution_context.ConfigureResource(
 			sol,
 			res,
-			knowledgebase.Configuration{Field: v.Ref.Property, Value: v.Template.DefaultValue},
+			knowledgebase.Configuration{Field: v.Ref.Property, Value: defaultVal},
 			dynData,
 			"set",
 		)
@@ -182,12 +190,6 @@ func (v *propertyVertex) evaluateConstraints(sol solution_context.SolutionContex
 		if property == nil {
 			return fmt.Errorf("could not get property %s from resource %s", v.Ref.Property, res.ID)
 		}
-		switch setConstraint.Operator {
-		case constraints.AddConstraintOperator, constraints.RemoveConstraintOperator:
-			if property.IsPropertyTypeScalar() {
-				return fmt.Errorf("cannot add/remove to scalar property %s", v.Ref)
-			}
-		}
 		err = solution_context.ConfigureResource(
 			sol,
 			res,
@@ -228,7 +230,7 @@ func (v *propertyVertex) evaluateConstraints(sol solution_context.SolutionContex
 }
 
 func (v *propertyVertex) evaluateResourceOperational(sol solution_context.SolutionContext, res *construct.Resource) error {
-	if v.Template == nil || v.Template.OperationalRule == nil {
+	if v.Template == nil || v.Template.Details().OperationalRule == nil {
 		return nil
 	}
 
@@ -238,7 +240,7 @@ func (v *propertyVertex) evaluateResourceOperational(sol solution_context.Soluti
 		Data:     knowledgebase.DynamicValueData{Resource: res.ID},
 	}
 
-	err := opCtx.HandleOperationalRule(*v.Template.OperationalRule)
+	err := opCtx.HandlePropertyRule(*v.Template.Details().OperationalRule)
 	if err != nil {
 		return fmt.Errorf("could not apply operational rule for %s: %w", v.Ref, err)
 	}
@@ -282,21 +284,26 @@ func (v *propertyVertex) Ready(eval *Evaluator) (ReadyPriority, error) {
 		// wait until we have a template
 		return NotReadyMax, nil
 	}
-	if v.Template.OperationalRule != nil {
+	if v.Template.Details().OperationalRule != nil {
 		// operational rules should run as soon as possible to create any resources they need
 		return ReadyNow, nil
 	}
-	if strings.Contains(v.Template.Type, "list") || strings.Contains(v.Template.Type, "set") {
+	ptype := v.Template.Type()
+	if strings.HasPrefix(ptype, "list") || strings.HasPrefix(ptype, "set") {
 		// never sure when a list/set is ready - it'll just be appended to by edges through
 		// `v.EdgeRules`
 		return NotReadyHigh, nil
 	}
-	if strings.Contains(v.Template.Type, "map") && len(v.Template.Properties) == 0 {
+	if strings.HasPrefix(ptype, "map") && len(v.Template.SubProperties()) == 0 {
 		// maps without sub-properties (ie, not objects) are also appended to by edges
 		return NotReadyHigh, nil
 	}
 	// properties that have values set via edge rules dont' have default values
-	if v.Template.DefaultValue != nil {
+	defaultVal, err := v.Template.GetDefaultValue(solution_context.DynamicCtx(eval.Solution), knowledgebase.DynamicValueData{Resource: v.Ref.Resource})
+	if err != nil {
+		return NotReadyMid, nil
+	}
+	if defaultVal != nil {
 		return ReadyNow, nil
 	}
 	// for non-list/set types, once an edge is here to set the value, it can be run