Deployer types

README.md

@@ -21,7 +21,7 @@ This binary can then be used to run Arcaflow workflows.
 The simplest workflow is the example plugin workflow using the workflow schema version `v0.1.0`: (save it to workflow.yaml)
 
 ```yaml
-version: v0.1.0
+version: v0.2.0
 input:
   root: RootObject
   objects:
@@ -72,7 +72,7 @@ You can load this config by passing the `-config` flag to Arcaflow.
 
 ### Supported Workflow Schema Versions
 
-- v0.1.0
+- v0.2.0
 
 ## Deployer options
 

config/config.go

@@ -15,9 +15,9 @@ type Config struct {
 	// TypeHintPlugins holds a list of plugins that will be used when building a type hint (e.g. JSONSchema) file for
 	// workflows.
 	TypeHintPlugins []string `json:"plugins" yaml:"plugins"`
-	// LocalDeployer holds the configuration for executing plugins locally. This deployer is used to obtain the schema
+	// LocalDeployers holds the configuration for executing plugins locally. This deployer is used to obtain the schema
 	// from the plugins before executing them in a remote environment.
-	LocalDeployer any `json:"deployer" yaml:"deployer"`
+	LocalDeployers map[string]any `json:"deployers" yaml:"deployers"`
 	// Log configures logging for workflow runs.
 	Log log.Config `json:"log" yaml:"log"`
 	// StepOutputLogging allows logging of step output

config/load_test.go

@@ -18,8 +18,8 @@ var configLoadData = map[string]struct {
 		input: "",
 		expectedOutput: &config.Config{
 			TypeHintPlugins: nil,
-			LocalDeployer: map[string]any{
-				"type": "docker",
+			LocalDeployers: map[string]any{
+				"image": map[string]string{"deployer_name": "docker"},
 			},
 			Log: log.Config{
 				Level:       log.LevelInfo,
@@ -34,8 +34,8 @@ log:
 `,
 		expectedOutput: &config.Config{
 			TypeHintPlugins: nil,
-			LocalDeployer: map[string]any{
-				"type": "docker",
+			LocalDeployers: map[string]any{
+				"image": map[string]string{"deployer_name": "docker"},
 			},
 			Log: log.Config{
 				Level:       log.LevelDebug,
@@ -45,13 +45,14 @@ log:
 	},
 	"type-kubernetes": {
 		input: `
-deployer:
-  type: kubernetes
+deployers:
+  image: 
+    deployer_name: kubernetes
 `,
 		expectedOutput: &config.Config{
 			TypeHintPlugins: nil,
-			LocalDeployer: map[string]any{
-				"type": "kubernetes",
+			LocalDeployers: map[string]any{
+				"image": map[string]string{"deployer_name": "kubernetes"},
 			},
 			Log: log.Config{
 				Level:       log.LevelInfo,
@@ -68,8 +69,8 @@ plugins:
 			TypeHintPlugins: []string{
 				"quay.io/arcalot/example-plugin:latest",
 			},
-			LocalDeployer: map[string]any{
-				"type": "docker",
+			LocalDeployers: map[string]any{
+				"image": map[string]string{"deployer_name": "docker"},
 			},
 			Log: log.Config{
 				Level:       log.LevelInfo,

config/schema.go

@@ -45,20 +45,25 @@ func getConfigSchema() *schema.TypedScopeSchema[*Config] {
 					nil,
 					nil,
 				),
-				"deployer": schema.NewPropertySchema(
-					schema.NewAnySchema(),
+				"deployers": schema.NewPropertySchema(
+					schema.NewMapSchema(
+						schema.NewStringSchema(nil, nil, nil),
+						schema.NewAnySchema(),
+						nil,
+						nil,
+					),
 					schema.NewDisplayValue(
-						schema.PointerTo("Local deployer"),
+						schema.PointerTo("Local deployers"),
 						schema.PointerTo(
-							"Local container environment configuration the workflow engine can use to test-deploy plugins before the workflow execution.",
+							"Default deployers for each plugin type.",
 						),
 						nil,
 					),
-					true,
+					false,
 					nil,
 					nil,
 					nil,
-					schema.PointerTo("{\"type\":\"docker\"}"),
+					schema.PointerTo(`{"image": {"deployer_name": "docker"}}`),
 					nil,
 				),
 				"logged_outputs": schema.NewPropertySchema(

engine.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	log "go.arcalot.io/log/v2"
-	"go.flow.arcalot.io/deployer/registry"
 	"go.flow.arcalot.io/engine/config"
 	"go.flow.arcalot.io/engine/internal/step"
 	"go.flow.arcalot.io/engine/internal/yaml"
@@ -14,7 +13,7 @@ import (
 )
 
 var supportedVersions = map[string]struct{}{
-	"v0.1.0": {},
+	"v0.2.0": {},
 }
 
 // WorkflowEngine is responsible for executing workflows and returning their result.
@@ -58,10 +57,9 @@ type Workflow interface {
 }
 
 type workflowEngine struct {
-	logger           log.Logger
-	deployerRegistry registry.Registry
-	stepRegistry     step.Registry
-	config           *config.Config
+	logger       log.Logger
+	stepRegistry step.Registry
+	config       *config.Config
 }
 
 func (w workflowEngine) RunWorkflow(ctx context.Context, input []byte, workflowContext map[string][]byte, workflowFileName string) (outputID string, outputData any, outputError bool, err error) {

engine_test.go

@@ -14,9 +14,11 @@ import (
 )
 
 func TestEngineWorkflow_ParseVersion(t *testing.T) {
-	_, err := engine.SupportedVersion("v0.1.0")
+	_, err := engine.SupportedVersion("v0.2.0")
 	assert.NoError(t, err)
-	_, err = engine.SupportedVersion("v0.11.0")
+
+	// test unsupported version
+	_, err = engine.SupportedVersion("v0.1000.0")
 	assert.Error(t, err)
 }
 
@@ -102,7 +104,7 @@ func TestEmptySteps(t *testing.T) {
 		context.Background(),
 		nil,
 		map[string][]byte{
-			"workflow.yaml": []byte(`version: v0.1.0
+			"workflow.yaml": []byte(`version: v0.2.0
 output: []
 steps: []`),
 		},
@@ -117,7 +119,7 @@ func TestNoSteps(t *testing.T) {
 		context.Background(),
 		nil,
 		map[string][]byte{
-			"workflow.yaml": []byte(`version: v0.1.0
+			"workflow.yaml": []byte(`version: v0.2.0
 output: []`),
 		},
 		"",
@@ -131,7 +133,7 @@ func TestE2E(t *testing.T) {
 		context.Background(),
 		[]byte(`name: Arca Lot`),
 		map[string][]byte{
-			"workflow.yaml": []byte(`version: v0.1.0
+			"workflow.yaml": []byte(`version: v0.2.0
 input:
   root: RootObject
   objects:
@@ -143,7 +145,9 @@ input:
             type_id: string
 steps:
   example:
-    plugin: quay.io/arcalot/arcaflow-plugin-template-python:0.2.1
+    plugin: 
+      src: quay.io/arcalot/arcaflow-plugin-template-python:0.2.1
+      deployment_type: image
     input:
       name: !expr $.input.name
 output:
@@ -162,7 +166,7 @@ func TestE2EMultipleOutputs(t *testing.T) {
 		context.Background(),
 		[]byte(`name: Arca Lot`),
 		map[string][]byte{
-			"workflow.yaml": []byte(`version: v0.1.0
+			"workflow.yaml": []byte(`version: v0.2.0
 input:
   root: RootObject
   objects:
@@ -174,7 +178,9 @@ input:
             type_id: string
 steps:
   example:
-    plugin: quay.io/arcalot/arcaflow-plugin-template-python:0.2.1
+    plugin: 
+      src: quay.io/arcalot/arcaflow-plugin-template-python:0.2.1
+      deployment_type: image
     input:
       name: !expr $.input.name
 outputs:

go.mod

@@ -7,14 +7,14 @@ require (
 	go.arcalot.io/dgraph v1.1.0
 	go.arcalot.io/lang v1.0.0
 	go.arcalot.io/log/v2 v2.0.0
-	go.flow.arcalot.io/deployer v0.3.0
-	go.flow.arcalot.io/dockerdeployer v0.4.0
+	go.flow.arcalot.io/deployer v0.4.0
+	go.flow.arcalot.io/dockerdeployer v0.5.0
 	go.flow.arcalot.io/expressions v0.2.1
-	go.flow.arcalot.io/kubernetesdeployer v0.7.0
+	go.flow.arcalot.io/kubernetesdeployer v0.8.0
 	go.flow.arcalot.io/pluginsdk v0.5.0
-	go.flow.arcalot.io/podmandeployer v0.5.0
-	go.flow.arcalot.io/pythondeployer v0.3.0
-	go.flow.arcalot.io/testdeployer v0.3.0
+	go.flow.arcalot.io/podmandeployer v0.6.0
+	go.flow.arcalot.io/pythondeployer v0.4.0
+	go.flow.arcalot.io/testdeployer v0.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -24,7 +24,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
-	github.com/docker/docker v24.0.6+incompatible // indirect
+	github.com/docker/docker v24.0.7+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.0 // indirect
@@ -50,7 +50,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	go.flow.arcalot.io/testplugin v0.2.1 // indirect
+	go.flow.arcalot.io/testplugin v0.3.0 // indirect
 	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/oauth2 v0.2.0 // indirect

go.sum

@@ -23,8 +23,8 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
-github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -160,24 +160,24 @@ go.arcalot.io/lang v1.0.0 h1:mgDaieT4wWdZTnR4V7+/pgYRmzfU7VZZgIzHccuxAbY=
 go.arcalot.io/lang v1.0.0/go.mod h1:ALqfYEhAzC2WoGLaycmJoNJd5NmkR7V1PSKp/c5D278=
 go.arcalot.io/log/v2 v2.0.0 h1:mbmsWDVBXZNWrDzUh5JLzeGCQ59kTuMFs+pyfJGc1hk=
 go.arcalot.io/log/v2 v2.0.0/go.mod h1:1V8jnFIIGwh2CtcGkHNOmy1nCo7LbazQNkUcnKYNMn4=
-go.flow.arcalot.io/deployer v0.3.0 h1:LPikgRG5jGA76W8JthycvzfREL5Y0+++KAiQxSnKhdU=
-go.flow.arcalot.io/deployer v0.3.0/go.mod h1:x6gsz/hANR8qN1nerpyY3vXpdaqofDH5Wlg+Nsqg/x0=
-go.flow.arcalot.io/dockerdeployer v0.4.0 h1:t5b8o3xfKKb/WIX558486csjo4uMQmAXsikBLsKFEIg=
-go.flow.arcalot.io/dockerdeployer v0.4.0/go.mod h1:UZSM6buJBRlgCURUE/BVkak8tfAXzj3oeQBSRZECbSc=
+go.flow.arcalot.io/deployer v0.4.0 h1:5YveLCX+zc8Ra/aukHOwD5OrJD2W8WRzoruf3bpJfqY=
+go.flow.arcalot.io/deployer v0.4.0/go.mod h1:x6gsz/hANR8qN1nerpyY3vXpdaqofDH5Wlg+Nsqg/x0=
+go.flow.arcalot.io/dockerdeployer v0.5.0 h1:CKsy5K38/0hF6dEry7f0YIfI5uNKGImyCwi8miwPEhE=
+go.flow.arcalot.io/dockerdeployer v0.5.0/go.mod h1:zE/okmoc47PTdJxiag+Gk9LMzKR1GemeP1Q3woOY3/A=
 go.flow.arcalot.io/expressions v0.2.1 h1:TAAbDrgJJLpmgA5ASyP/KzrXWtpEaQ8JsCPHgpe5kLw=
 go.flow.arcalot.io/expressions v0.2.1/go.mod h1:Vw1ScNu4Uyw1/l87LAH8jxe0DyRWwMh+rlfB/BPYDOU=
-go.flow.arcalot.io/kubernetesdeployer v0.7.0 h1:r41qWc/XiPy9l3cfMXZG8F2kGenRh1xsx2auim/Ydyw=
-go.flow.arcalot.io/kubernetesdeployer v0.7.0/go.mod h1:VvU6duoo5NR2ITUhx/UCGrkdJnXIeYm+/yHmGKtkXsk=
+go.flow.arcalot.io/kubernetesdeployer v0.8.0 h1:UjH/aspPif/k+X65sLWlNDZAW5JlzUfgOnLHOrhxEQk=
+go.flow.arcalot.io/kubernetesdeployer v0.8.0/go.mod h1:BhERhKpvQMJkrcW9lbBF4kJEe+OGhz2NpSftZIgtVNQ=
 go.flow.arcalot.io/pluginsdk v0.5.0 h1:TRS/waCTcdoMZ9neDAcfy3zpzyDnPHRbhV+Y1kpcw3Y=
 go.flow.arcalot.io/pluginsdk v0.5.0/go.mod h1:2s2f//7uOkBjr1QaiWJD/bqDIeLlINJtD1BhiY4aGPM=
-go.flow.arcalot.io/podmandeployer v0.5.0 h1:h7hEhWUgxJzNKlEohZ+meKhl3FWjaXQahQ8vN3YVRNs=
-go.flow.arcalot.io/podmandeployer v0.5.0/go.mod h1:36JCcTB6nauahcXUPfIpdEw7Zfp0ufM07o3VNTvrCc0=
-go.flow.arcalot.io/pythondeployer v0.3.0 h1:ercLuDwFoDSL0f6YvZEqFW0/nO7Yv7DkbROl3rKxYDk=
-go.flow.arcalot.io/pythondeployer v0.3.0/go.mod h1:ND1x/Vhu/6q50zQeisCcD6oQ6lKVJFflOrfDccnIjSY=
-go.flow.arcalot.io/testdeployer v0.3.0 h1:Soyz+rDa3Y3VjWBGuL3zNlX3LM4uKp9Ex7///fCgrZA=
-go.flow.arcalot.io/testdeployer v0.3.0/go.mod h1:Eel0ORhtKdYYDsd+e+btBBygIn+9Sz/b+JFDwH39VWI=
-go.flow.arcalot.io/testplugin v0.2.1 h1:9kQ2MKvcXtEcwk5c4qSWN+FovpER2C9vn730laAm9iE=
-go.flow.arcalot.io/testplugin v0.2.1/go.mod h1:ZoVF8tIKppQmj5nvoZPA48GQ7BuoWXQcuCw2x2sJxjE=
+go.flow.arcalot.io/podmandeployer v0.6.0 h1:SlcQUU6xt24Oa0OFNnwYE+d+XQbDXErMLvpCi2gMHoA=
+go.flow.arcalot.io/podmandeployer v0.6.0/go.mod h1:4wfcl0qjV02y64We3ZSDz+3lwdOfbe+gpFjm7SQKTRA=
+go.flow.arcalot.io/pythondeployer v0.4.0 h1:l8nw6awYMVzgND+ZXdbnNJPYu3V0sgSUFsIzn+SRgh0=
+go.flow.arcalot.io/pythondeployer v0.4.0/go.mod h1:me9SIMVTCBzCmceILdBMxXYrZGakiPOMasHgujmgJlE=
+go.flow.arcalot.io/testdeployer v0.4.0 h1:helexgZOnYlbzU+egkxsLs95iMwE3lc+vp+Qbv2xwFI=
+go.flow.arcalot.io/testdeployer v0.4.0/go.mod h1:Luw76oKeri40a7k8buk7Q7J86Tpt8lf9LTVZr6lqgTk=
+go.flow.arcalot.io/testplugin v0.3.0 h1:LY0VWL1M0X+mSE0nVCwoD0PjQHqKhbka2Gc70uuP04k=
+go.flow.arcalot.io/testplugin v0.3.0/go.mod h1:TW6h/kZyM+4gBBKqvoDI0XWXgNcUMKbDHaQOaKIRZdk=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

internal/step/foreach/provider_example_test.go

@@ -16,7 +16,7 @@ import (
 
 // mainWorkflow is the workflow calling the foreach step.
 var mainWorkflow = `
-version: v0.1.0
+version: v0.2.0
 input:
   root: names
   objects:
@@ -44,7 +44,7 @@ output:
 `
 
 var subworkflow = `
-version: v0.1.0
+version: v0.2.0
 input:
   root: name
   objects: