From 3815fc3b77f013d4c22f574a2d5caa5e45bc4448 Mon Sep 17 00:00:00 2001 From: Matt Leader Date: Tue, 17 Oct 2023 12:24:38 -0400 Subject: [PATCH] Add Version to Workflow Specification (#112) * add api version to model and validate supported version * refactor to use version in workflow spec * fix other broken tests --- engine.go | 31 ++++++++++++++++--- engine_test.go | 19 +++++++++--- go.mod | 12 +++---- go.sum | 26 ++++++++-------- .../step/foreach/provider_example_test.go | 2 ++ workflow/executor_example_test.go | 1 + workflow/executor_test.go | 1 + workflow/model.go | 21 ++++++++++++- workflow/workflow_test.go | 4 +++ 9 files changed, 88 insertions(+), 29 deletions(-) diff --git a/engine.go b/engine.go index b5dd6f53..df3dd536 100644 --- a/engine.go +++ b/engine.go @@ -4,17 +4,19 @@ package engine import ( "context" "fmt" - log "go.arcalot.io/log/v2" - "go.flow.arcalot.io/engine/internal/step" - "go.flow.arcalot.io/engine/workflow" - "go.flow.arcalot.io/pluginsdk/schema" - "go.flow.arcalot.io/deployer/registry" "go.flow.arcalot.io/engine/config" + "go.flow.arcalot.io/engine/internal/step" "go.flow.arcalot.io/engine/internal/yaml" + "go.flow.arcalot.io/engine/workflow" + "go.flow.arcalot.io/pluginsdk/schema" ) +var supportedVersions = map[string]struct{}{ + "v0.1.0": {}, +} + // WorkflowEngine is responsible for executing workflows and returning their result. type WorkflowEngine interface { // RunWorkflow is a simplified shortcut to parse and immediately run a workflow. @@ -88,6 +90,12 @@ func (w workflowEngine) Parse( return nil, err } + v, err := SupportedVersion(wf.Version) + if err != nil { + return nil, err + } + wf.Version = v + executor, err := workflow.NewExecutor(w.logger, w.config, w.stepRegistry) if err != nil { return nil, err @@ -103,6 +111,19 @@ func (w workflowEngine) Parse( }, nil } +// SupportedVersion confirms whether a given version string +// is in the set of supported workflow specifications. It +// returns true when the version is in the set, false otherwise. +// Earlier schema validation already applies version's +// regular expression. +func SupportedVersion(version string) (string, error) { + _, ok := supportedVersions[version] + if !ok { + return version, fmt.Errorf("unsupported workflow schema version: %s", version) + } + return version, nil +} + type engineWorkflow struct { workflow workflow.ExecutableWorkflow } diff --git a/engine_test.go b/engine_test.go index cfe4906d..bd5a45bf 100644 --- a/engine_test.go +++ b/engine_test.go @@ -13,6 +13,13 @@ import ( "go.flow.arcalot.io/engine/config" ) +func TestEngineWorkflow_ParseVersion(t *testing.T) { + _, err := engine.SupportedVersion("v0.1.0") + assert.NoError(t, err) + _, err = engine.SupportedVersion("v0.11.0") + assert.Error(t, err) +} + func createTestEngine(t *testing.T) engine.WorkflowEngine { cfg := config.Default() cfg.Log.T = t @@ -95,7 +102,8 @@ func TestEmptySteps(t *testing.T) { context.Background(), nil, map[string][]byte{ - "workflow.yaml": []byte(`output: [] + "workflow.yaml": []byte(`version: v0.1.0 +output: [] steps: []`), }, "", @@ -109,7 +117,8 @@ func TestNoSteps(t *testing.T) { context.Background(), nil, map[string][]byte{ - "workflow.yaml": []byte(`output: []`), + "workflow.yaml": []byte(`version: v0.1.0 +output: []`), }, "", ) @@ -122,7 +131,8 @@ func TestE2E(t *testing.T) { context.Background(), []byte(`name: Arca Lot`), map[string][]byte{ - "workflow.yaml": []byte(`input: + "workflow.yaml": []byte(`version: v0.1.0 +input: root: RootObject objects: RootObject: @@ -152,7 +162,8 @@ func TestE2EMultipleOutputs(t *testing.T) { context.Background(), []byte(`name: Arca Lot`), map[string][]byte{ - "workflow.yaml": []byte(`input: + "workflow.yaml": []byte(`version: v0.1.0 +input: root: RootObject objects: RootObject: diff --git a/go.mod b/go.mod index 68d3a95e..97cb47c1 100644 --- a/go.mod +++ b/go.mod @@ -49,14 +49,14 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.flow.arcalot.io/testplugin v0.1.0 // indirect - golang.org/x/mod v0.11.0 // indirect - golang.org/x/net v0.11.0 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.15.0 // indirect golang.org/x/oauth2 v0.2.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.2.0 // indirect - golang.org/x/tools v0.9.3 // indirect + golang.org/x/tools v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 963620b0..50b30625 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -199,8 +199,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.2.0 h1:GtQkldQ9m7yvzCL1V+LrYow3Khe0eJH0w7RbX/VbaIU= @@ -211,7 +211,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -222,18 +222,18 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -244,8 +244,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/step/foreach/provider_example_test.go b/internal/step/foreach/provider_example_test.go index 928a8e16..952cf7ae 100644 --- a/internal/step/foreach/provider_example_test.go +++ b/internal/step/foreach/provider_example_test.go @@ -16,6 +16,7 @@ import ( // mainWorkflow is the workflow calling the foreach step. var mainWorkflow = ` +version: v0.1.0 input: root: names objects: @@ -43,6 +44,7 @@ output: ` var subworkflow = ` +version: v0.1.0 input: root: name objects: diff --git a/workflow/executor_example_test.go b/workflow/executor_example_test.go index 25627206..25a1011f 100644 --- a/workflow/executor_example_test.go +++ b/workflow/executor_example_test.go @@ -13,6 +13,7 @@ import ( ) var workflowYAML = `--- +version: v0.1.0 input: root: RootObject objects: diff --git a/workflow/executor_test.go b/workflow/executor_test.go index 059f195c..cec272dd 100644 --- a/workflow/executor_test.go +++ b/workflow/executor_test.go @@ -14,6 +14,7 @@ import ( ) var sharedInputWorkflowYAML = `--- +version: v0.1.0 input: root: RootObject objects: diff --git a/workflow/model.go b/workflow/model.go index a3534e93..f5ff010c 100644 --- a/workflow/model.go +++ b/workflow/model.go @@ -12,6 +12,8 @@ import ( // Workflow is the primary data structure describing workflows. type Workflow struct { + // Version determines which set of the arcaflow workflow external interface will be used in the workflow. + Version string `json:"version"` // Input describe the input schema for a workflow. These values can be referenced from expressions. The structure // must be a scope described in primitive types. This is done so later on a forward reference to a step input can // be used. @@ -26,7 +28,7 @@ type Workflow struct { // expressions. The keys must be the output IDs from Outputs and the values must be a StepOutputSchema object as // per the Arcaflow schema. OutputSchema map[string]any `json:"outputSchema"` - // Output is the legay way to define a single output. It conflicts the "outputs" field and if filled, will create a + // Output is the legacy way to define a single output. It conflicts the "outputs" field and if filled, will create a // "success" output. // // Deprecated: use Outputs instead. @@ -39,6 +41,23 @@ func getSchema() *schema.TypedScopeSchema[*Workflow] { schema.NewStructMappedObjectSchema[*Workflow]( "Workflow", map[string]*schema.PropertySchema{ + "version": schema.NewPropertySchema( + schema.NewStringSchema( + schema.IntPointer(1), + schema.IntPointer(255), + regexp.MustCompile(`^v\d+\.\d+\.\d+$`)), + schema.NewDisplayValue( + schema.PointerTo("Version"), + schema.PointerTo("Arcaflow Workflow specification version to be used."), + nil, + ), + true, + nil, + nil, + nil, + nil, + nil, + ), "input": schema.NewPropertySchema( schema.NewAnySchema(), schema.NewDisplayValue( diff --git a/workflow/workflow_test.go b/workflow/workflow_test.go index fb81dc37..7faff9de 100644 --- a/workflow/workflow_test.go +++ b/workflow/workflow_test.go @@ -22,6 +22,7 @@ import ( ) var badWorkflowDefinition = ` +version: v0.1.0 input: root: name objects: @@ -70,6 +71,7 @@ func TestOutputFailed(t *testing.T) { } var stepCancellationWorkflowDefinition = ` +version: v0.1.0 input: root: RootObject objects: @@ -148,6 +150,7 @@ func TestStepCancellation(t *testing.T) { } var waitForSerialWorkflowDefinition = ` +version: v0.1.0 input: root: RootObject objects: @@ -228,6 +231,7 @@ func TestWaitForSerial(t *testing.T) { // Running parallel steps which wait on the same previous step sometimes causes a race condition. This needs to be investigated. // once the race condition if fixed reduce the wait_time to 500ms. var waitForParallelWorkflowDefinition = ` +version: v0.1.0 input: root: RootObject objects: