From 5dbf988cb4c6a73e62d78644524c9809677d965c Mon Sep 17 00:00:00 2001 From: Matt Leader Date: Fri, 26 Jan 2024 12:31:23 -0500 Subject: [PATCH] Support N Depth Sub-Workflow References (#146) * refactor and add tests * add newline --- engine.go | 22 +++++++++++++-- ...orkflow-1.yaml => depth-1_workflow-1.yaml} | 0 .../test-subworkflow/depth-1_workflow-2.yaml | 28 +++++++++++++++++++ .../test-subworkflow/depth-2_workflow-1.yaml | 28 +++++++++++++++++++ ...orkflow-2.yaml => depth-3_workflow-1.yaml} | 0 fixtures/test-subworkflow/workflow-happy.yaml | 6 ++-- loadfile/loadfile.go | 3 ++ loadfile/loadfile_test.go | 7 +++++ 8 files changed, 89 insertions(+), 5 deletions(-) rename fixtures/test-subworkflow/{subworkflow-1.yaml => depth-1_workflow-1.yaml} (100%) create mode 100644 fixtures/test-subworkflow/depth-1_workflow-2.yaml create mode 100644 fixtures/test-subworkflow/depth-2_workflow-1.yaml rename fixtures/test-subworkflow/{subworkflow-2.yaml => depth-3_workflow-1.yaml} (100%) diff --git a/engine.go b/engine.go index 392f239e..61ed73ae 100644 --- a/engine.go +++ b/engine.go @@ -94,7 +94,8 @@ func (w workflowEngine) Parse( return nil, err } - stepWorkflowFileCache, err := SubworkflowCache(wf, files.RootDir()) + flowCaches := make([]loadfile.FileCache, 0) + stepWorkflowFileCache, err := SubworkflowCache(wf, files.RootDir(), yamlConverter, flowCaches) if err != nil { return nil, err } @@ -150,7 +151,7 @@ func StepWorkflowPaths(wf *workflow.Workflow) map[string]string { // SubworkflowCache creates a file cache of the sub-workflows referenced // in this workflow using rootDir as a context. -func SubworkflowCache(wf *workflow.Workflow, rootDir string) (loadfile.FileCache, error) { +func SubworkflowCache(wf *workflow.Workflow, rootDir string, converter workflow.YAMLConverter, flowCaches []loadfile.FileCache) (loadfile.FileCache, error) { stepWorkflowPaths := StepWorkflowPaths(wf) if len(stepWorkflowPaths) == 0 { return nil, nil @@ -163,6 +164,23 @@ func SubworkflowCache(wf *workflow.Workflow, rootDir string) (loadfile.FileCache if err != nil { return nil, err } + for _, ctxFile := range subworkflowCache.Files() { + subwf, err := converter.FromYAML(ctxFile.Content) + if err != nil { + return nil, err + } + flowCache, err := SubworkflowCache(subwf, rootDir, converter, flowCaches) + if err != nil { + return nil, err + } + flowCaches = append(flowCaches, flowCache) + } + + flowCaches = append(flowCaches, subworkflowCache) + subworkflowCache, err = loadfile.MergeFileCaches(flowCaches...) + if err != nil { + return nil, err + } return subworkflowCache, nil } diff --git a/fixtures/test-subworkflow/subworkflow-1.yaml b/fixtures/test-subworkflow/depth-1_workflow-1.yaml similarity index 100% rename from fixtures/test-subworkflow/subworkflow-1.yaml rename to fixtures/test-subworkflow/depth-1_workflow-1.yaml diff --git a/fixtures/test-subworkflow/depth-1_workflow-2.yaml b/fixtures/test-subworkflow/depth-1_workflow-2.yaml new file mode 100644 index 00000000..fb304f8f --- /dev/null +++ b/fixtures/test-subworkflow/depth-1_workflow-2.yaml @@ -0,0 +1,28 @@ +version: v0.2.0 +input: + root: RootObject + objects: + RootObject: + id: RootObject + properties: + seconds: + type: + type_id: float + min: 0 +steps: + wait_1: + plugin: + src: quay.io/arcalot/arcaflow-plugin-wait:atp-v3_cc3e02b + deployment_type: image + step: wait + input: + seconds: !expr $.input.seconds + wait_loop: + kind: foreach + items: + - seconds: .1 + workflow: depth-2_workflow-1.yaml +outputs: + success: + a: !expr $.steps.wait_1.outputs + b: !expr $.steps.wait_loop.outputs diff --git a/fixtures/test-subworkflow/depth-2_workflow-1.yaml b/fixtures/test-subworkflow/depth-2_workflow-1.yaml new file mode 100644 index 00000000..2a9de0e3 --- /dev/null +++ b/fixtures/test-subworkflow/depth-2_workflow-1.yaml @@ -0,0 +1,28 @@ +version: v0.2.0 +input: + root: RootObject + objects: + RootObject: + id: RootObject + properties: + seconds: + type: + type_id: float + min: 0 +steps: + wait_1: + plugin: + src: quay.io/arcalot/arcaflow-plugin-wait:atp-v3_cc3e02b + deployment_type: image + step: wait + input: + seconds: !expr $.input.seconds + wait_loop: + kind: foreach + items: + - seconds: .1 + workflow: depth-3_workflow-1.yaml +outputs: + success: + a: !expr $.steps.wait_1.outputs + b: !expr $.steps.wait_loop.outputs diff --git a/fixtures/test-subworkflow/subworkflow-2.yaml b/fixtures/test-subworkflow/depth-3_workflow-1.yaml similarity index 100% rename from fixtures/test-subworkflow/subworkflow-2.yaml rename to fixtures/test-subworkflow/depth-3_workflow-1.yaml diff --git a/fixtures/test-subworkflow/workflow-happy.yaml b/fixtures/test-subworkflow/workflow-happy.yaml index 3a559cbd..2d259925 100644 --- a/fixtures/test-subworkflow/workflow-happy.yaml +++ b/fixtures/test-subworkflow/workflow-happy.yaml @@ -10,17 +10,17 @@ steps: kind: foreach items: - seconds: .1 - workflow: subworkflow-1.yaml + workflow: depth-1_workflow-1.yaml wait_wf_2: kind: foreach items: - seconds: .1 - workflow: subworkflow-2.yaml + workflow: depth-1_workflow-2.yaml wait_wf_3: kind: foreach items: - seconds: .1 - workflow: subworkflow-1.yaml + workflow: depth-1_workflow-1.yaml outputs: success: step_1: !expr $.steps.wait_wf_1.outputs diff --git a/loadfile/loadfile.go b/loadfile/loadfile.go index da725cfe..4aa3539a 100644 --- a/loadfile/loadfile.go +++ b/loadfile/loadfile.go @@ -158,6 +158,9 @@ func MergeFileCaches(fileCaches ...FileCache) (FileCache, error) { rootDir := "" for _, fc := range fileCaches { + if fc == nil { + continue + } for key, contextFile := range fc.Files() { cache[key] = contextFile } diff --git a/loadfile/loadfile_test.go b/loadfile/loadfile_test.go index d36443c9..756776f3 100644 --- a/loadfile/loadfile_test.go +++ b/loadfile/loadfile_test.go @@ -172,6 +172,13 @@ func Test_MergeFileCaches(t *testing.T) { fcMerged2, err := loadfile.MergeFileCaches(fcMerged, fc3) assert.Error(t, err) assert.Nil(t, fcMerged2) + + // nil file caches should not be merged + var fcNil loadfile.FileCache + fcMerged3, err := loadfile.MergeFileCaches(fcMerged, fcNil, fcNil) + assert.NoError(t, err) + assert.Equals(t, fcMerged3.RootDir(), expRootDir) + assert.Equals(t, fcMerged3.Files(), expMergedFiles) } func FileCacheAbsPaths(fc loadfile.FileCache) map[string]string {