diff --git a/cmd/cwl2argo/transpiler/ast_cl.go b/cmd/cwl2argo/transpiler/ast_cl.go new file mode 100644 index 000000000000..0380e42ff0ea --- /dev/null +++ b/cmd/cwl2argo/transpiler/ast_cl.go @@ -0,0 +1,303 @@ +package transpiler + +// interface due to large number of child types +type CWLRequirements interface { + isCWLRequirement() +} + +type DockerRequirement struct { + Class string `yaml:"class"` + DockerPull *string `yaml:"dockerPull"` + DockerLoad *string `yaml:"dockerLoad"` + DockerFile *string `yaml:"dockerFile"` + DockerImport *string `yaml:"dockerImport"` + DockerImageId *string `yaml:"dockerImageId"` + DockerOutputDirectory *string `yaml:"dockerOutputDirectory"` +} + +type SoftwarePackage struct { + Package string `yaml:"package"` + Version Strings `yaml:"version"` + Specs Strings `yaml:"specs"` +} + +type SoftwareRequirement struct { + Class string `yaml:"class"` // constant SoftwareRequirement + Packages []SoftwarePackage `yaml:"packages"` +} + +type LoadListingRequirement struct { + Class string `yaml:"class"` // constant LoadListingRequirement + LoadListing *LoadListingEnum `yaml:"loadListing"` +} + +type Dirent struct { + entry CWLExpression `yaml:"entry"` + entryName *CWLExpression `yaml:"entryName"` + writeable *bool `yaml:"writeable"` +} + +type InitialWorkDirRequirementListing interface { + isInitialWorkDirRequirementListing() +} + +type InitialWorkDirRequirement struct { + Class string `yaml:"class"` // constant InitialWorkDirRequirement + Listing InitialWorkDirRequirementListing `yaml:"listing"` +} + +type InlineJavascriptRequirement struct { + Class string `yaml:"class"` // constant InlineJavascriptRequirement + ExpressionLib Strings `yaml:"expressionLib"` +} + +type SchemaDefRequirementType interface { + isSchemaDefRequirementType() +} + +type SchemaDefRequirement struct { + Class string `yaml:"class"` // constant SchemaDefRequirement + Types []SchemaDefRequirementType `yaml:"types"` +} + +type EnvironmentDef struct { + EnvName string `yaml:"envName"` + EnvValue CWLExpression `yaml:"envValue"` +} + +type EnvVarRequirement struct { + Class string `yaml:"class"` // constant EnvVarRequirement + EnvDef []EnvironmentDef `yaml:"envDef"` +} + +type ShellCommandRequirement struct { + Class string `yaml:"class"` // constant ShellCommandRequirement +} + +type WorkReuse struct { + Class string `yaml:"class"` // constant WorkReuse + EnableReuse CWLExpression `yaml:"enableReuse"` +} + +type NetworkAccess struct { + Class string // constant NetworkAccess + NetworkAccess CWLExpression +} + +type InplaceUpdateRequirement struct { + Class string `yaml:"class"` // constant InplaceUpdateRequirement + InplaceUpdate Bool `yaml:"inplaceUpdate"` +} + +type ToolTimeLimit struct { + Class string `yaml:"class"` // constant ToolTimeLimit + TimeLimit CWLExpression `yaml:"timeLimit"` +} + +type ResourceRequirement struct { + Class string `yaml:"class"` // constand ResourceRequirement + CoresMin *CWLExpression `yaml:"coresMin"` + CoresMax *CWLExpression `yaml:"coresMax"` + RamMin *CWLExpression `yaml:"ramMin"` + RamMax *CWLExpression `yaml:"ramMax"` + TmpdirMin *CWLExpression `yaml:"tmpdirMin"` + TmpdirMax *CWLExpression `yaml:"tmpdirMin"` + OutdirMin *CWLExpression `yaml:"outdirMin"` + OutdirMax *CWLExpression `yaml:"outdirMax"` +} + +func (_ InlineJavascriptRequirement) isCWLRequirement() {} +func (_ SchemaDefRequirement) isCWLRequirement() {} +func (_ LoadListingRequirement) isCWLRequirement() {} +func (_ DockerRequirement) isCWLRequirement() {} +func (_ SoftwareRequirement) isCWLRequirement() {} +func (_ InitialWorkDirRequirement) isCWLRequirement() {} +func (_ EnvVarRequirement) isCWLRequirement() {} +func (_ ShellCommandRequirement) isCWLRequirement() {} +func (_ WorkReuse) isCWLRequirement() {} +func (_ NetworkAccess) isCWLRequirement() {} +func (_ InplaceUpdateRequirement) isCWLRequirement() {} +func (_ ToolTimeLimit) isCWLRequirement() {} + +func (_ CommandlineInputRecordSchema) isSchemaDefRequirementType() {} +func (_ CommandlineInputEnumSchema) isSchemaDefRequirementType() {} +func (_ CommandlineInputArraySchema) isSchemaDefRequirementType() {} +func (_ DockerRequirement) isSchemaDefRequirementType() {} +func (_ SoftwareRequirement) isSchemaDefRequirementType() {} +func (_ InitialWorkDirRequirement) isSchemaDefRequirementType() {} + +type CommandlineInputRecordField struct { + Name string `yaml:"name"` + Type CommandlineTypes `yaml:"type"` // len(1) represents scalar len > 1 represents array + Doc Strings `yaml:"doc"` + Label *string `yaml:"label"` + SecondaryFiles SecondaryFiles `yaml:"secondaryFiles"` + Streamable *bool `yaml:"streamable"` + Format CWLFormat `yaml:"format"` + LoadContents *bool `yaml:"loadContents"` + LoadListing LoadListingEnum `yaml:"loadListing"` + InputBinding *CommandlineBinding `yaml:"inputBinding"` +} + +type CommandlineInputArraySchema struct { + Items CommandlineTypes `yaml:"items"` + Type string `yaml:"type"` // MUST be array + Label *string `yaml:"label"` + Doc Strings `yaml:"doc"` + Name *string `yaml:"name"` + InputBinding *CommandlineBinding `yaml:"inputBinding"` +} + +type CommandlineInputEnumSchema struct { + Symbols Strings `yaml:"symbols"` + Type string `yaml:"type"` // MUST BE enum, only a placeholder for type verification purposes + Label *string `yaml:"label"` + Doc Strings `yaml:"doc"` + Name *string `yaml:"name"` + InputBinding *CommandlineBinding +} + +type CommandlineInputRecordSchema struct { + Type string `yaml:"type"` // MUST BE "record" + Fields *[]CommandlineInputRecordField `yaml:"fields"` + Label *string `yaml:"label"` + Doc *Strings `yaml:"doc"` + Name *string `yaml:"name"` + inputBinding *CommandlineBinding `yaml:"inputBinding"` +} + +func (_ CWLNull) isFlat() {} +func (_ CWLBool) isFlat() {} +func (_ CWLInt) isFlat() {} +func (_ CWLLong) isFlat() {} +func (_ CWLFloat) isFlat() {} +func (_ CWLDouble) isFlat() {} +func (_ CWLFile) isFlat() {} +func (_ CWLDirectory) isFlat() {} +func (_ CWLStdin) isFlat() {} +func (_ String) isFlat() {} +func (_ CommandlineInputEnumSchema) isFlat() {} + +type Type int32 + +const ( + CWLNullKind Type = iota + CWLBoolKind + CWLIntKind + CWLLongKind + CWLFloatKind + CWLDoubleKind + CWLFileKind + CWLDirectoryKind + CWLStdinKind + CWLStringKind + CWLRecordKind + CWLRecordFieldKind + CWLEnumKind + CWLArrayKind +) + +type CommandlineType struct { + Kind Type + Record *CommandlineInputRecordSchema + Enum *CommandlineInputEnumSchema + Array *CommandlineInputArraySchema +} + +type CommandlineTypes []CommandlineType + +type CommandlineBinding struct { + LoadContents *bool `yaml:"loadContents"` + Position *int `yaml:"position"` + Prefix *string `yaml:"prefix"` + Seperate *bool `yaml:"seperate"` + ItemSeperator *string `yaml:"itemSeperator"` + ValueFrom CWLExpression `yaml:"valueFrom"` + ShellQuote *bool `yaml:"bool"` +} + +type CommandlineInputParameter struct { + Type CommandlineTypes `yaml:"type"` // len(1) == scalar while len > 1 == array + Label *string `yaml:"label"` + SecondaryFiles SecondaryFiles `yaml:"secondaryFiles"` // len(1) == scalar while len > 1 == array + Streamable *bool `yaml:"streamable"` + Doc Strings `yaml:"doc"` + Id *string `yaml:"id"` + Format CWLFormat `yaml:"format"` + LoadContents *bool `yaml:"loadContents"` + LoadListing LoadListingEnum `yaml:"loadListing"` + Default interface{} `yaml:"default"` + InputBinding *CommandlineBinding `yaml:"inputBinding"` +} + +type OutputBindingGlobKind int32 + +const ( + GlobStringKind OutputBindingGlobKind = iota + GlobStringsKind + GlobExpressionKind +) + +type CommandlineOutputBindingGlob struct { + Kind OutputBindingGlobKind + String String + Strings Strings + Expression CWLExpression +} + +type CommandlineOutputBinding struct { + LoadContents *bool `yaml:"loadContents"` + LoadListing LoadListingEnum `yaml:"loadListing"` + Glob CommandlineOutputBindingGlob `yaml:"glob"` + OutputEval CWLExpression `yaml:"outputEval"` +} + +type CommandlineOutputParameter struct { + Type CommandlineTypes `yaml:"type"` + Label *string `yaml:"label"` + SecondaryFiles SecondaryFiles `yaml:"secondaryFiles"` + Streamable *bool `yaml:"streamable"` + Doc Strings `yaml:"doc"` + Id *string `yaml:"id"` + Format CWLFormat `yaml:"format"` + OutputBinding *CommandlineOutputBinding `yaml:"outputBinding"` +} + +type CommandlineArgumentKind int32 + +const ( + ArguementStringKind CommandlineArgumentKind = iota + ArguementExpressionKind + ArguementCLIBindingKind +) + +type CommandlineArgument struct { + Kind CommandlineArgumentKind + String String + Expression CWLExpression + CommandlineBinding CommandlineBinding +} + +type Inputs []CommandlineInputParameter +type Outputs []CommandlineOutputParameter +type Requirements []CWLRequirements +type Hints []interface{} +type Arguments []CommandlineArgument + +type CommandlineTool struct { + Inputs Inputs `yaml:"inputs"` + Outputs Outputs `yaml:"outputs"` + Class string `yaml:"class"` // Must be "CommandLineTool" + Id *string `yaml:"id"` + Label *string `yaml:"label"` + Doc Strings `yaml:"doc"` + Requirements Requirements `yaml:"requirements"` + Hints Hints `yaml:"hints"` + CWLVersion *string `yaml:"cwlVersion"` + Intent Strings `yaml:"intent"` + BaseCommand Strings `yaml:"baseCommand"` + Arguments Arguments `yaml:"arguments"` + Stdin CWLExpression `yaml:"stdin"` + Stderr CWLExpression `yaml:"stderr"` + Stdout CWLExpression `yaml:"stdout"` +} diff --git a/cmd/cwl2argo/transpiler/ast_common.go b/cmd/cwl2argo/transpiler/ast_common.go new file mode 100644 index 000000000000..d8e6747b0bcf --- /dev/null +++ b/cmd/cwl2argo/transpiler/ast_common.go @@ -0,0 +1,89 @@ +package transpiler + +type ( + String string + Bool bool + Int int + Float float32 + Strings []string + SecondaryFiles []CWLSecondaryFileSchema +) + +type CWLFormatKind int32 + +const ( + FormatStringKind CWLFormatKind = iota + FormatStringsKind + FormatExpressionKind +) + +type CWLFormat struct { + Kind CWLFormatKind + String String + Strings Strings + Expression CWLExpression +} + +type ( + CWLNull struct{} + CWLBool struct{} + CWLInt struct{} + CWLLong struct{} + CWLFloat struct{} + CWLDouble struct{} + CWLString struct{} + CWLFile struct{} + CWLDirectory struct{} +) + +func (_ CWLNull) isCWLType() {} +func (_ CWLBool) isCWLType() {} +func (_ CWLInt) isCWLType() {} +func (_ CWLLong) isCWLType() {} +func (_ CWLFloat) isCWLType() {} +func (_ CWLDouble) isCWLType() {} +func (_ CWLString) isCWLType() {} +func (_ CWLFile) isCWLType() {} +func (_ CWLDirectory) isCWLType() {} + +type CWLStdin struct{} + +type CWLType interface { + isCWLType() +} + +type LoadListingKind int32 + +const ( + ShallowListingKind LoadListingKind = iota + DeepListingKind + NoListingKind +) + +type LoadListingEnum struct { + Kind LoadListingKind +} + +type CWLExpressionKind int32 + +const ( + RawKind CWLExpressionKind = iota + ExpressionKind + BoolKind + IntKind + FloatKind +) + +type CWLExpression struct { + Kind CWLExpressionKind + Raw string + Expression string + Bool bool + Int int + Float float64 +} + +type CWLSecondaryFileSchema struct { + Pattern CWLExpression `yaml:"pattern"` + Required CWLExpression `yaml:"required"` +} diff --git a/cmd/cwl2argo/transpiler/fill_cl.go b/cmd/cwl2argo/transpiler/fill_cl.go new file mode 100644 index 000000000000..c4a11e921b49 --- /dev/null +++ b/cmd/cwl2argo/transpiler/fill_cl.go @@ -0,0 +1,303 @@ +package transpiler + +import ( + "errors" + "fmt" + + "gopkg.in/yaml.v3" +) + +func keyNotPresentError(key string) error { + return fmt.Errorf("Could not find %s", key) +} + +// intermediate representation used to +// parse into interfaces. The class string is used +// to decode Node into a structure. +type IntermediateRepr struct { + Class *string + Node *yaml.Node +} + +func getCWLExpressionInner(str string) *string { + if len(str) < 3 { + return nil + } + if str[0] != '$' { + return nil + } + if str[1] == '[' && str[len(str)-1] == ']' { + return &str + } + if str[1] == '{' && str[len(str)-1] == '}' { + return &str + } + return nil +} + +func (s *Strings) UnmarshalYAML(value *yaml.Node) error { + strings := make([]string, 0) + switch value.Kind { + case yaml.ScalarNode: + var s string + if err := value.Decode(&s); err != nil { + return err + } + strings = append(strings, s) + case yaml.SequenceNode: + if err := value.Decode(&strings); err != nil { + return err + } + default: + return errors.New("string | []string expected") + } + *s = strings + return nil +} + +func (tys *CommandlineTypes) UnmarshalYAML(value *yaml.Node) error { + newTys := make([]CommandlineType, 0) + switch value.Kind { + case yaml.ScalarNode: + var s string + if err := value.Decode(&s); err != nil { + return err + } + var ty CommandlineType + switch s { + case "string": + ty.Kind = CWLStringKind + case "null": + ty.Kind = CWLNullKind + case "boolean": + ty.Kind = CWLBoolKind + case "int": + ty.Kind = CWLIntKind + case "long": + ty.Kind = CWLLongKind + case "float": + ty.Kind = CWLFloatKind + case "double": + ty.Kind = CWLDoubleKind + case "File": + ty.Kind = CWLFileKind + case "Directory": + ty.Kind = CWLDirectoryKind + default: + return fmt.Errorf("%s is not a supported type", s) + } + newTys = append(newTys, ty) + case yaml.MappingNode: + return errors.New("complex types not supported yet") + case yaml.SequenceNode: + return errors.New("array types not supported yet") + default: + return errors.New("type not supported") + } + *tys = newTys + return nil +} + +func (format *CWLFormat) UnmarshalYAML(value *yaml.Node) error { + switch value.Kind { + case yaml.ScalarNode: + var s string + if err := value.Decode(&s); err != nil { + return err + } + format.Kind = FormatStringKind + format.String = String(s) + return nil + case yaml.SequenceNode: + s := make([]string, 0) + if err := value.Decode(&s); err != nil { + return err + } + format.Kind = FormatStringsKind + format.Strings = s + return nil + default: + return errors.New("string | []string expected") + } +} + +func (listing *LoadListingEnum) UnmarshalYAML(value *yaml.Node) error { + return errors.New("LoadListingEnum not yet supported") +} + +func (input *CommandlineInputParameter) UnmarshalYAML(value *yaml.Node) error { + type rawParamType CommandlineInputParameter + + err := value.Decode((*rawParamType)(input)) + if err != nil { + return err + } + return nil +} + +func (inp *Inputs) UnmarshalYAML(value *yaml.Node) error { + inputs := make([]CommandlineInputParameter, 0) + + switch value.Kind { + case yaml.MappingNode: + m := make(map[string]CommandlineInputParameter) + err := value.Decode(&m) + if err != nil { + return err + } + for key, input := range m { + input.Id = &key + inputs = append(inputs, input) + } + case yaml.SequenceNode: + err := value.Decode(&inputs) + if err != nil { + return err + } + default: + return errors.New("sequence or mapping type expected") + } + *inp = inputs + return nil +} + +func (out *Outputs) UnmarshalYAML(value *yaml.Node) error { + outputs := make([]CommandlineOutputParameter, 0) + + switch value.Kind { + case yaml.MappingNode: + m := make(map[string]CommandlineOutputParameter) + err := value.Decode(&m) + if err != nil { + return err + } + for key, output := range m { + output.Id = &key + outputs = append(outputs, output) + } + case yaml.SequenceNode: + err := value.Decode(&outputs) + if err != nil { + return err + } + default: + return errors.New("Sequence or mapping type expected") + } + + *out = outputs + + return nil +} + +func (ir *IntermediateRepr) UnmarshalYAML(value *yaml.Node) error { + m := make(map[string]interface{}) + err := value.Decode(&m) + if err != nil { + return err + } + classi, ok := m["class"] + if ok { + class, ok := classi.(string) + if !ok { + return errors.New("string expected") + } + ir.Class = &class + } + ir.Node = value + return nil +} + +func (reqs *Requirements) UnmarshalYAML(value *yaml.Node) error { + rs := make(map[string]IntermediateRepr, 0) + err := value.Decode(&rs) + if err != nil { + rsArray := make([]IntermediateRepr, 0) + err = value.Decode(&rsArray) + if err != nil { + return errors.New("[]requirement or map[class]requirement was expected") + } + for _, req := range rsArray { + if req.Class == nil { + return errors.New("class expected") + } + rs[*req.Class] = req + } + } + + newRequests := make([]CWLRequirements, 0) + for class, req := range rs { + switch class { + case "DockerRequirement": + var d DockerRequirement + err := req.Node.Decode(&d) + if err != nil { + return err + } + newRequests = append(newRequests, d) + default: + return fmt.Errorf("%s is not implemented", class) + } + } + *reqs = newRequests + return nil +} + +func (hints Hints) UnmarshalYAML(value *yaml.Node) error { + return errors.New("hints not supported yet") +} + +func (args Arguments) UnmarshalYAML(value *yaml.Node) error { + return errors.New("arguments not supported yet") +} + +func (expr *CWLExpression) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.ScalarNode { + return errors.New("can only be string | bool | int | float") + } + var f float64 + err := value.Decode(&f) + if err == nil { + expr.Kind = FloatKind + expr.Float = f + return nil + } + + var i int + err = value.Decode(&i) + if err == nil { + expr.Kind = IntKind + expr.Int = i + return nil + } + + var b bool + err = value.Decode(&b) + if err == nil { + expr.Kind = BoolKind + expr.Bool = b + return nil + } + + var s string + err = value.Decode(&s) + if err == nil { + exprS := getCWLExpressionInner(s) + if exprS != nil { + expr.Kind = ExpressionKind + expr.Expression = *exprS + return nil + } + expr.Kind = RawKind + expr.Raw = s + return nil + } + return errors.New("can only be string | bool | int | float") +} + +func (cl *CommandlineTool) UnmarshalYAML(value *yaml.Node) error { + type rawCLITool CommandlineTool + if err := value.Decode((*rawCLITool)(cl)); err != nil { + return err + } + return nil +} diff --git a/cmd/cwl2argo/transpiler/fill_cl_test.go b/cmd/cwl2argo/transpiler/fill_cl_test.go new file mode 100644 index 000000000000..43d2a75576b6 --- /dev/null +++ b/cmd/cwl2argo/transpiler/fill_cl_test.go @@ -0,0 +1,31 @@ +package transpiler + +import ( + "testing" + + "gopkg.in/yaml.v3" +) + +func TestSimpleCWL(t *testing.T) { + data := `cwlVersion: v1.2 +class: CommandLineTool +requirements: + - class: DockerRequirement + dockerPull: python:3.7 +baseCommand: echo +id: echo-tool +inputs: + message: + type: string + inputBinding: + position: 1 +outputs: [] +` + + var cliTool CommandlineTool + err := yaml.Unmarshal([]byte(data), &cliTool) + if err != nil { + t.Error(err) + } + +}