Multi file support

README.md

@@ -1,4 +1,41 @@
-# proteus
+# Proteus
 
-Proteus is a tool for transpiling the common workflow language into the argo workflow syntax
+Proteus is a tool for transpiling resources from the Common Workflow Language into resources types compatible with Argo Workflows. 
 
+The input and output of Proteus is only files. The output file should represent a valid Argo Workflow resource, which can then be parsed directly into Argo Workflows.
+
+![alt text](cwl2argo.png)
+
+## Installation
+
+The tool is available as a command line interface that can be downloaded to Linux, Max, and Windows.
+
+#### On Linux
+```
+# Curl the binary
+curl -sLO https://github.com/SerRichard/proteus/releases/download/<version>/proteus-linux-amd64.gz
+
+# Unzip
+gunzip proteus-linux-amd64.gz
+
+# Make executable
+chmod +x proteus-linux-amd64
+
+# Move binary
+mv ./proteus-linux-amd64 /usr/local/bin/proteus
+
+# Check installation
+proteus --help
+```
+
+Executables are also available with each release for Windows and Mac, they are available for download under the following names. Installation instructions for each will to follow.
+
+#### For Mac
+```
+proteus-darwin-amd64.gz
+```
+
+#### For Windows
+```
+proteus-windows-amd64
+```

pkg/cwl/ast_common.go

@@ -90,6 +90,7 @@ type CWLInputEntry struct {
 	BoolData   *bool
 	StringData *string
 	IntData    *int
+	Array      *[]any
 }
 
 type LoadListingEnum string
@@ -147,6 +148,14 @@ func (cwlInputEntry *CWLInputEntry) UnmarshalYAML(value *yaml.Node) error {
 		return nil
 	}
 
+	var arr []any
+	err = value.Decode(&arr)
+	if err == nil {
+		cwlInputEntry.Kind = CWLArrayKind
+		cwlInputEntry.Array = &arr
+		return nil
+	}
+
 	var file CWLFile
 	err = value.Decode(&file)
 	if err == nil {

pkg/cwl/fill_cl.go

@@ -3,6 +3,7 @@ package cwl
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
@@ -62,6 +63,11 @@ func (tys *CWLTypes) UnmarshalYAML(value *yaml.Node) error {
 			return err
 		}
 		var ty CWLType
+
+		if strings.Contains(s, "[]") {
+			s = "array"
+		}
+
 		switch s {
 		case "string":
 			ty.Kind = CWLStringKind
@@ -81,11 +87,14 @@ func (tys *CWLTypes) UnmarshalYAML(value *yaml.Node) error {
 			ty.Kind = CWLFileKind
 		case "Directory":
 			ty.Kind = CWLDirectoryKind
+		case "array":
+			ty.Kind = CWLArrayKind
 		default:
 			return fmt.Errorf("%s is not a supported type", s)
 		}
 		newTys = append(newTys, ty)
 	case yaml.MappingNode:
+		fmt.Println("MappingNode ", value)
 		return errors.New("complex types not supported yet")
 	case yaml.SequenceNode:
 		return errors.New("array types not supported yet")

pkg/transpiler/emit_cli.go

@@ -27,7 +27,9 @@ type flatCommandlineInputParameter struct {
 	Type             cwl.Type
 	Label            *string
 	StringValue      *string               // string value
+	BoolValue        *bool                 // boolean value
 	IntValue         *int                  // int value
+	ArrayValue       *[]any                // Array value
 	Emit             bool                  // boolean value
 	File             *cwl.CWLFile          // file value
 	FileLocationData *cwl.FileLocationData // file location data
@@ -95,13 +97,11 @@ func dockerNotPresent() error              { return errors.New("DockerRequiremen
 func resourceRequirementNotPresent() error { return errors.New("ResourceRequirement was not found") }
 
 func findDockerRequirement(requirements cwl.Requirements) (*cwl.DockerRequirement, error) {
-	log.Info("Need DockerRequirement")
 	var docker *cwl.DockerRequirement
 	docker = nil
 	for _, req := range requirements {
 		d, ok := req.(cwl.DockerRequirement)
 		if ok {
-			log.Info("Found DockerRequirement")
 			docker = &d
 		}
 	}
@@ -115,11 +115,11 @@ func findDockerRequirement(requirements cwl.Requirements) (*cwl.DockerRequiremen
 
 func findResourceRequirement(requirements cwl.Requirements) (*cwl.ResourceRequirement, error) {
 	var resource *cwl.ResourceRequirement
+
 	resource = nil
 	for _, req := range requirements {
 		r, ok := req.(cwl.ResourceRequirement)
 		if ok {
-			log.Info("Found ResourceRequirement")
 			resource = &r
 		}
 	}
@@ -169,7 +169,6 @@ func (inputParameter CommandlineInputParameter) getInputBindings(inputs map[stri
 	}
 
 	if len(inputs) == 0 {
-
 		switch v := inputParameter.CommandlineInputParameter.Default.(type) {
 		case string:
 			binding.StringValue = &v
@@ -196,6 +195,10 @@ func (inputParameter CommandlineInputParameter) getInputBindings(inputs map[stri
 		binding.IntValue = input.IntData
 	case cwl.CWLFileKind:
 		binding.File = input.FileData
+	case cwl.CWLBoolKind:
+		binding.BoolValue = input.BoolData
+	case cwl.CWLArrayKind:
+		binding.ArrayValue = input.Array
 	default:
 		return nil, fmt.Errorf("%T unknown type", input.Kind)
 	}
@@ -304,6 +307,7 @@ func emitArgumentParams(container *apiv1.Container,
 				continue
 			}
 			if binding.File == nil || binding.File.Path == nil {
+				log.Info("binding, ", binding.File)
 				return errors.New("file information was not available")
 			}
 			arg = *binding.File.Path
@@ -329,7 +333,11 @@ func emitArguments(spec *v1alpha1.WorkflowSpec, bindings []flatCommandlineInputP
 		case cwl.CWLIntKind:
 			intString := fmt.Sprintf("%d", *binding.IntValue)
 			params = append(params, v1alpha1.Parameter{Name: *binding.Id, Value: (*v1alpha1.AnyString)(&intString)})
+		case cwl.CWLBoolKind:
+			boolString := fmt.Sprintf("%t", *binding.BoolValue)
+			params = append(params, v1alpha1.Parameter{Name: *binding.Id, Value: (*v1alpha1.AnyString)(&boolString)})
 		default:
+			log.Info("HERE ", binding.Type)
 			return fmt.Errorf("%T is not supported", binding.Type)
 		}
 	}
@@ -443,6 +451,11 @@ func emitPVC(spec *v1alpha1.WorkflowSpec, resourceReq *cwl.ResourceRequirement)
 func emitInputArtifacts(template *v1alpha1.Template, inputs map[string]cwl.CWLInputEntry, locations cwl.FileLocations) error {
 	arts := make([]v1alpha1.Artifact, 0)
 
+	// If there are no locations, do not try to infer an artifact should exist.
+	if len(locations.Inputs) == 0 {
+		return nil
+	}
+
 	for key, inputEntry := range inputs {
 		if inputEntry.Kind != cwl.CWLFileKind {
 			continue
@@ -477,6 +490,12 @@ func evalCommandlineBindingOutputGlob(bglob *cwl.CommandlineOutputBindingGlob) (
 }
 
 func emitOutputArtifact(tmpl *v1alpha1.Template, output flatCommandlineOutputParameter, locations cwl.FileLocations) error {
+
+	// If there are no locations, do not try to infer an artifact should exist.
+	if len(locations.Outputs) == 0 {
+		return nil
+	}
+
 	if output.Type != cwl.CWLFileKind {
 		return errors.New("emitOutputArtifact only accepts CWLFileKind")
 	}
@@ -485,6 +504,7 @@ func emitOutputArtifact(tmpl *v1alpha1.Template, output flatCommandlineOutputPar
 	if err != nil {
 		return err
 	}
+
 	location, ok := locations.Outputs[*output.Id]
 	if !ok {
 		return fmt.Errorf("unable to find output for %s", *output.Id)
@@ -563,6 +583,8 @@ func EmitCommandlineTool(clTool *cwl.CommandLineTool, inputs map[string]cwl.CWLI
 		return nil, err
 	}
 
+	log.Info("Need pvc? ", outputBindings)
+
 	if needPVC(outputBindings) {
 
 		resourceRequirement, err := findResourceRequirement(clTool.Requirements)

test/data/composite-cli/array-inputs/array-inputs-job.yml

@@ -0,0 +1,3 @@
+filesA: [one, two, three]
+filesB: [four, five, six]
+filesC: [seven, eight, nine]

test/data/composite-cli/array-inputs/array-inputs.cwl

@@ -0,0 +1,37 @@
+cwlVersion: v1.2
+class: CommandLineTool
+id: array-inputs
+requirements: 
+  - class: DockerRequirement 
+    dockerPull: ubuntu:20.04
+    dockerOutputDirectory: /tmp
+
+  - class: ResourceRequirement 
+    outdirMin: 1Gi
+
+inputs:
+  filesA:
+    type: string[]
+    inputBinding:
+      prefix: -A
+      position: 1
+
+  filesB:
+    type: array
+    inputBinding:
+      position: 2
+
+  filesC:
+    type: string[]
+    inputBinding:
+      prefix: -C=
+      itemSeparator: ","
+      separate: false
+      position: 4
+
+outputs:
+  example_out:
+    type: File
+    outputSource: tmp/stdout_data
+
+baseCommand: echo

test/data/composite-cli/array-inputs/array-inputs_argo_output.yaml

@@ -0,0 +1,42 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+    creationTimestamp: null
+    name: array-inputs
+spec:
+    arguments: {}
+    entrypoint: array-inputs
+    templates:
+        - container:
+            args:
+                - -A
+                - '{{inputs.parameters.filesA}}'
+                - '{{inputs.parameters.filesB}}'
+                - -C={{inputs.parameters.filesC}}
+            command:
+                - echo
+            image: ubuntu:20.04
+            name: ""
+            resources: {}
+            volumeMounts:
+                - mountPath: /tmp
+                  name: argovolume
+            workingDir: /tmp
+          inputs: {}
+          metadata: {}
+          name: array-inputs
+          outputs: {}
+    volumeClaimTemplates:
+        - metadata:
+            creationTimestamp: null
+            name: argovolume
+          spec:
+            accessModes:
+                - ReadWriteMany
+            resources:
+                requests:
+                    storage: 1Gi
+          status: {}
+status:
+    finishedAt: null
+    startedAt: null

test/data/composite-cli/inputs/inp-job.yml

@@ -0,0 +1,6 @@
+example_flag: true
+example_string: hello
+example_int: 42
+example_file:
+  class: File
+  path: /tmp/whale.txt

test/data/composite-cli/inputs/inp.cwl

@@ -0,0 +1,32 @@
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: echo
+inputs:
+  example_flag:
+    type: boolean
+    inputBinding:
+      position: 1
+      prefix: -f
+  example_string:
+    type: string
+    inputBinding:
+      position: 3
+      prefix: --example-string
+  example_int:
+    type: int
+    inputBinding:
+      position: 2
+      prefix: -i
+      separate: false
+  example_file:
+    type: File
+    inputBinding:
+      prefix: --file=
+      separate: false
+      position: 4
+outputs: []
+id: echo-tool
+requirements: 
+  - class: DockerRequirement 
+    dockerPull: ubuntu:20.04
+    dockerOutputDirectory: /tmp

test/data/composite-cli/param-ref/tar_param.cwl

@@ -0,0 +1,21 @@
+cwlVersion: v1.2
+class: CommandLineTool
+id: param-ref
+requirements: 
+  - class: DockerRequirement 
+    dockerPull: ubuntu:20.04
+    dockerOutputDirectory: /tmp
+
+  - class: ResourceRequirement 
+    outdirMin: 1Gi
+baseCommand: [tar, --extract]
+inputs:
+  tarfile:
+    type: File
+    inputBinding:
+      prefix: --file
+outputs:
+  extracted_file:
+    type: File
+    outputBinding:
+      glob: $(inputs.extractfile)

test/data/composite-cli/param-ref/tar_param_job.yml

@@ -0,0 +1,4 @@
+tarfile:
+  class: File
+  path: hello.tar
+extractfile: goodbye.txt

test/data/composite-cli/whalesay/inputs.yaml

@@ -0,0 +1 @@
+message: Hello, CWL!

test/data/composite-cli/whalesay/print-message.cwl

@@ -0,0 +1,14 @@
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: [cowsay]
+inputs:
+  message:
+    type: string
+    inputBinding:
+      position: 1
+outputs: []
+requirements:
+  - class: DockerRequirement
+    dockerPull: docker/whalesay:latest
+    dockerLoad: true
+arguments: ["$(inputs.message)"]