Upgrade SDK to use ApplyNamespace instead of ApplyScope (#183)

* Upgade SDK to use ApplyNamespace instad of ApplyScope

* Embed schema.AnySchema type in anySchemaWithExpressions

* Defer error logic to schema.AnySchema

go.mod

@@ -13,7 +13,7 @@ require (
 	go.flow.arcalot.io/dockerdeployer v0.7.0
 	go.flow.arcalot.io/expressions v0.4.2
 	go.flow.arcalot.io/kubernetesdeployer v0.9.1
-	go.flow.arcalot.io/pluginsdk v0.11.1
+	go.flow.arcalot.io/pluginsdk v0.12.0
 	go.flow.arcalot.io/podmandeployer v0.11.0
 	go.flow.arcalot.io/pythondeployer v0.6.0
 	go.flow.arcalot.io/testdeployer v0.6.0

go.sum

@@ -143,8 +143,8 @@ go.flow.arcalot.io/expressions v0.4.2 h1:nEfxpWGEG1+59CoiAnS3AUQ+j2RZcUSPhPGjwPd
 go.flow.arcalot.io/expressions v0.4.2/go.mod h1:FA/50wX1+0iTgW/dFeeE1yOslZSmfBaMNR4IiMYRwxc=
 go.flow.arcalot.io/kubernetesdeployer v0.9.1 h1:AGnJFazehAENXxGMCF0Uc7aG9F0LpvuhoyQFu8deJG0=
 go.flow.arcalot.io/kubernetesdeployer v0.9.1/go.mod h1:yvxT3VwmyrlIi4422pxl02z4QeU2Gvbjg5aQB17Ye4s=
-go.flow.arcalot.io/pluginsdk v0.11.1 h1:vutbhJVSqCqVvgYHNTEYa0Hx0p+lxh9HoyFhhC+ui94=
-go.flow.arcalot.io/pluginsdk v0.11.1/go.mod h1:7HafTRTFTYRbJ4sS/Vn0CFrHlaBpEoyOX4oNf612XJM=
+go.flow.arcalot.io/pluginsdk v0.12.0 h1:NJyPMluwtljsNjEEewCeuxE2dGuAJn89KvU52o9OAvU=
+go.flow.arcalot.io/pluginsdk v0.12.0/go.mod h1:7HafTRTFTYRbJ4sS/Vn0CFrHlaBpEoyOX4oNf612XJM=
 go.flow.arcalot.io/podmandeployer v0.11.0 h1:U44YwzIoR3Tpm5ePDVQhycx3ALeu1AYiLxcdQbSvNEQ=
 go.flow.arcalot.io/podmandeployer v0.11.0/go.mod h1:gy+xL1oyyMKkdPUpkPFm94B3E2WySaqMl9sMmdrMk+4=
 go.flow.arcalot.io/pythondeployer v0.6.0 h1:ptAurEJ2u2U127nK6Kk7zTelbkk6ipPqZcwnTmqB9vo=

internal/util/invalid_serialization_detector.go

@@ -19,6 +19,7 @@ func NewInvalidSerializationDetectorSchema() *InvalidSerializationDetectorSchema
 // the data has been serialized or unserialized at least once (since, if it is
 // never operated on, then it's trivial to claim that it was never doubly done).
 type InvalidSerializationDetectorSchema struct {
+	schema.ScalarType
 	SerializeCnt   int
 	UnserializeCnt int
 }
@@ -89,15 +90,6 @@ func (d *InvalidSerializationDetectorSchema) SerializeType(data string) (any, er
 	return d.Serialize(data)
 }
 
-// ApplyScope is for applying a scope to the references. Does not apply to this object.
-func (d *InvalidSerializationDetectorSchema) ApplyScope(_ schema.Scope, _ string) {}
-
-// ValidateReferences validates that all necessary references from scopes have been applied.
-// Does not apply to this object.
-func (d *InvalidSerializationDetectorSchema) ValidateReferences() error {
-	return nil
-}
-
 // TypeID returns the category of type this type is. Returns string because
 // the valid states of this type include all strings.
 func (d *InvalidSerializationDetectorSchema) TypeID() schema.TypeID {

workflow/any.go

@@ -17,11 +17,7 @@ func newAnySchemaWithExpressions() *anySchemaWithExpressions {
 // anySchemaWithExpressions is a wildcard allowing maps, lists, integers, strings, bools, and floats. It also allows
 // expression objects.
 type anySchemaWithExpressions struct {
-	anySchema schema.AnySchema
-}
-
-func (a *anySchemaWithExpressions) ReflectedType() reflect.Type {
-	return a.anySchema.ReflectedType()
+	schema.AnySchema
 }
 
 func (a *anySchemaWithExpressions) Unserialize(data any) (any, error) {
@@ -39,59 +35,19 @@ func (a *anySchemaWithExpressions) ValidateCompatibility(dataOrType any) error {
 		// Assume okay
 		return nil
 	}
-	return a.anySchema.ValidateCompatibility(dataOrType)
+	return a.AnySchema.ValidateCompatibility(dataOrType)
 }
 
 func (a *anySchemaWithExpressions) Serialize(data any) (any, error) {
 	return a.checkAndConvert(data)
 }
 
-func (a *anySchemaWithExpressions) ApplyScope(scope schema.Scope, namespace string) {
-	a.anySchema.ApplyScope(scope, namespace)
-}
-
-func (a *anySchemaWithExpressions) ValidateReferences() error {
-	return a.anySchema.ValidateReferences()
-}
-
-func (a *anySchemaWithExpressions) TypeID() schema.TypeID {
-	return a.anySchema.TypeID()
-}
-
 func (a *anySchemaWithExpressions) checkAndConvert(data any) (any, error) {
 	if _, ok := data.(expressions.Expression); ok {
 		return data, nil
 	}
 	t := reflect.ValueOf(data)
 	switch t.Kind() {
-	case reflect.Int:
-		fallthrough
-	case reflect.Uint:
-		fallthrough
-	case reflect.Int8:
-		fallthrough
-	case reflect.Uint8:
-		fallthrough
-	case reflect.Int16:
-		fallthrough
-	case reflect.Uint16:
-		fallthrough
-	case reflect.Int32:
-		fallthrough
-	case reflect.Uint32:
-		fallthrough
-	case reflect.Uint64:
-		fallthrough
-	case reflect.Int64:
-		fallthrough
-	case reflect.Float32:
-		fallthrough
-	case reflect.Float64:
-		fallthrough
-	case reflect.String:
-		fallthrough
-	case reflect.Bool:
-		return a.anySchema.Unserialize(data)
 	case reflect.Slice:
 		result := make([]any, t.Len())
 		for i := 0; i < t.Len(); i++ {
@@ -118,8 +74,6 @@ func (a *anySchemaWithExpressions) checkAndConvert(data any) (any, error) {
 		}
 		return result, nil
 	default:
-		return nil, &schema.ConstraintError{
-			Message: fmt.Sprintf("unsupported data type for 'any' type: %T", data),
-		}
+		return a.AnySchema.Unserialize(data)
 	}
 }

workflow/executor.go

@@ -227,7 +227,7 @@ func (e *executor) processInput(workflow *Workflow) (schema.Scope, error) {
 	if !ok {
 		return nil, fmt.Errorf("bug: unserialized input is not a scope")
 	}
-	typedInput.ApplyScope(typedInput, schema.DEFAULT_NAMESPACE)
+	typedInput.ApplySelf()
 	return typedInput, nil
 }
 
@@ -317,7 +317,7 @@ func applyLifecycleScopes(
 	stepLifecycles map[string]step.Lifecycle[step.LifecycleStageWithSchema],
 	typedInput schema.Scope,
 ) error {
-	allNamespaces := make(map[string]schema.Scope)
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
 	for workflowStepID, stepLifecycle := range stepLifecycles {
 		for _, stage := range stepLifecycle.Stages {
 			prefix := "$.steps." + workflowStepID + "." + stage.ID + "."
@@ -335,19 +335,19 @@ func applyLifecycleScopes(
 // applyAllNamespaces applies all namespaces to the given scope.
 // This function also validates references, and lists the namespaces
 // and their objects in the event of a reference failure.
-func applyAllNamespaces(allNamespaces map[string]schema.Scope, scopeToApplyTo schema.Scope) error {
-	for namespace, scope := range allNamespaces {
-		scopeToApplyTo.ApplyScope(scope, namespace)
+func applyAllNamespaces(allNamespaces map[string]map[string]*schema.ObjectSchema, scopeToApplyTo schema.Scope) error {
+	for namespace, objects := range allNamespaces {
+		scopeToApplyTo.ApplyNamespace(objects, namespace)
 	}
 	err := scopeToApplyTo.ValidateReferences()
 	if err == nil {
 		return nil // Success
 	}
 	// Now on the error path. Provide useful debug info.
 	availableObjects := ""
-	for namespace, scope := range allNamespaces {
+	for namespace, objects := range allNamespaces {
 		availableObjects += "\n\t" + namespace + ":"
-		for objectID := range scope.Objects() {
+		for objectID := range objects {
 			availableObjects += " " + objectID
 		}
 	}
@@ -359,13 +359,13 @@ func applyAllNamespaces(allNamespaces map[string]schema.Scope, scopeToApplyTo sc
 	)
 }
 
-func addOutputNamespacedScopes(allNamespaces map[string]schema.Scope, stage step.LifecycleStageWithSchema, prefix string) {
+func addOutputNamespacedScopes(allNamespaces map[string]map[string]*schema.ObjectSchema, stage step.LifecycleStageWithSchema, prefix string) {
 	for outputID, outputSchema := range stage.Outputs {
 		addScopesWithReferences(allNamespaces, outputSchema.Schema(), prefix+outputID)
 	}
 }
 
-func addInputNamespacedScopes(allNamespaces map[string]schema.Scope, stage step.LifecycleStageWithSchema, prefix string) {
+func addInputNamespacedScopes(allNamespaces map[string]map[string]*schema.ObjectSchema, stage step.LifecycleStageWithSchema, prefix string) {
 	for inputID, inputSchemaProperty := range stage.InputSchema {
 		var inputSchemaType = inputSchemaProperty.Type()
 		// Extract item values from lists (like for ForEach)
@@ -379,19 +379,19 @@ func addInputNamespacedScopes(allNamespaces map[string]schema.Scope, stage step.
 }
 
 // Adds the scope to the namespace map, as well as all resolved references from external namespaces.
-func addScopesWithReferences(allNamespaces map[string]schema.Scope, scope schema.Scope, prefix string) {
-	// First, just adds the scope
-	allNamespaces[prefix] = scope
+func addScopesWithReferences(allNamespaces map[string]map[string]*schema.ObjectSchema, scope schema.Scope, prefix string) {
+	// First, just adds the scope's objects.
+	allNamespaces[prefix] = scope.Objects()
 	// Next, checks all properties for resolved references that reference objects outside of this scope.
 	rootObject := scope.RootObject()
 	for propertyID, property := range rootObject.Properties() {
 		if property.Type().TypeID() == schema.TypeIDRef {
 			refProperty := property.Type().(schema.Ref)
-			if refProperty.Namespace() != schema.DEFAULT_NAMESPACE {
+			if refProperty.Namespace() != schema.SelfNamespace {
 				// Found a reference to an object that is not included in the scope. Add it to the map.
 				var referencedObject any = refProperty.GetObject()
 				refObjectSchema := referencedObject.(*schema.ObjectSchema)
-				allNamespaces[prefix+"."+propertyID] = schema.NewScopeSchema(refObjectSchema)
+				allNamespaces[prefix+"."+propertyID] = map[string]*schema.ObjectSchema{refObjectSchema.ID(): refObjectSchema}
 			}
 		}
 	}

workflow/executor_unit_test.go

@@ -59,17 +59,17 @@ var testLifecycleStage = step.LifecycleStageWithSchema{
 }
 
 func TestAddOutputNamespacedScopes(t *testing.T) {
-	allNamespaces := make(map[string]schema.Scope)
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
 	namespacePrefix := "TEST_PREFIX_"
 	addOutputNamespacedScopes(allNamespaces, testLifecycleStage, namespacePrefix)
 	expectedNamespace := namespacePrefix + "outputC"
 	assert.Equals(t, len(allNamespaces), 1)
 	assert.MapContainsKey(t, expectedNamespace, allNamespaces)
-	assert.Equals(t, "testObjectC", allNamespaces[expectedNamespace].Root())
+	assert.MapContainsKey(t, "testObjectC", allNamespaces[expectedNamespace])
 }
 
 func TestAddInputNamespacedScopes(t *testing.T) {
-	allNamespaces := make(map[string]schema.Scope)
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
 	namespacePrefix := "TEST_PREFIX_"
 	expectedInputs := map[string]string{
 		"scopeA": "testObjectA",
@@ -81,12 +81,12 @@ func TestAddInputNamespacedScopes(t *testing.T) {
 	for expectedInput, expectedObject := range expectedInputs {
 		expectedNamespace := namespacePrefix + expectedInput
 		assert.MapContainsKey(t, expectedNamespace, allNamespaces)
-		assert.Equals(t, expectedObject, allNamespaces[expectedNamespace].Root())
+		assert.MapContainsKey(t, expectedObject, allNamespaces[expectedNamespace])
 	}
 }
 
 func TestAddScopesWithMissingCache(t *testing.T) {
-	allNamespaces := make(map[string]schema.Scope)
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
 	externalRef3 := schema.NewNamespacedRefSchema("scopeTestObjectC", "not-applied-namespace", nil)
 	notAppliedExternalRefProperty := schema.NewPropertySchema(
 		externalRef3,
@@ -117,7 +117,7 @@ func TestAddScopesWithMissingCache(t *testing.T) {
 
 func TestAddScopesWithReferences(t *testing.T) {
 	// Test that the scope itself and the resolved references are added.
-	allNamespaces := make(map[string]schema.Scope)
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
 	internalRef := schema.NewRefSchema("scopeTestObjectA", nil)
 	internalRefProperty := schema.NewPropertySchema(
 		internalRef,
@@ -178,9 +178,9 @@ func TestAddScopesWithReferences(t *testing.T) {
 			"scopeTestObjectB", map[string]*schema.PropertySchema{},
 		),
 	)
-	testScope.ApplyScope(scopeToApply, "test-namespace-1")
-	testScope.ApplyScope(scopeToApply, "$.test-namespace-2")
-	testScope.ApplyScope(testScope, schema.DEFAULT_NAMESPACE)
+	testScope.ApplyNamespace(scopeToApply.Objects(), "test-namespace-1")
+	testScope.ApplyNamespace(scopeToApply.Objects(), "$.test-namespace-2")
+	testScope.ApplySelf()
 	addScopesWithReferences(allNamespaces, testScope, "$")
 	assert.Equals(t, len(allNamespaces), 3)
 	assert.MapContainsKey(t, "$", allNamespaces)
@@ -229,17 +229,18 @@ func TestApplyAllNamespaces_Pass(t *testing.T) {
 	assert.Panics(t, func() {
 		ref2Schema.GetObject()
 	})
-	allNamespaces := make(map[string]schema.Scope)
-	allNamespaces["test-namespace-1"] = schema.NewScopeSchema(
-		schema.NewObjectSchema(
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
+	// Test one manual map creation, and one using scope's `Objects()` method.
+	allNamespaces["test-namespace-1"] = map[string]*schema.ObjectSchema{
+		"scopeTestObjectB": schema.NewObjectSchema(
 			"scopeTestObjectB", map[string]*schema.PropertySchema{},
 		),
-	)
+	}
 	allNamespaces["test-namespace-2"] = schema.NewScopeSchema(
 		schema.NewObjectSchema(
 			"scopeTestObjectC", map[string]*schema.PropertySchema{},
 		),
-	)
+	).Objects()
 	// Call the function under test and validate that all caches are set.
 	assert.NoError(t, applyAllNamespaces(allNamespaces, testScope))
 	assert.NoError(t, testScope.ValidateReferences())
@@ -289,12 +290,12 @@ func TestApplyAllNamespaces_MissingNamespace(t *testing.T) {
 		ref2Schema.GetObject()
 	})
 	// Only add test-namespace-1
-	allNamespaces := make(map[string]schema.Scope)
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
 	allNamespaces["test-namespace-1"] = schema.NewScopeSchema(
 		schema.NewObjectSchema(
 			"scopeTestObjectB", map[string]*schema.PropertySchema{},
 		),
-	)
+	).Objects()
 	err := applyAllNamespaces(allNamespaces, testScope)
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "error validating references")
@@ -324,12 +325,12 @@ func TestApplyAllNamespaces_InvalidObject(t *testing.T) {
 			},
 		),
 	)
-	allNamespaces := make(map[string]schema.Scope)
+	allNamespaces := make(map[string]map[string]*schema.ObjectSchema)
 	allNamespaces["test-namespace-1"] = schema.NewScopeSchema(
 		schema.NewObjectSchema(
 			"scopeTestObjectB", map[string]*schema.PropertySchema{},
 		),
-	)
+	).Objects()
 	assert.PanicsContains(
 		t,
 		func() {