Add Hail batch + translator base refactor

janis_core/ingestion/fromwdl.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import re
 from types import LambdaType
@@ -153,6 +155,8 @@ def container_from_runtime(cls, runtime, inputs: List[WDL.Decl]):
     def parse_memory_requirement(self, value):
         s = self.translate_expr(value)
         if isinstance(s, str):
+            if s.lower().endswith("g"):
+                return float(s[:-1].strip())
             if s.lower().endswith("gb"):
                 return float(s[:-2].strip())
             elif s.lower().endswith("gib"):
@@ -315,6 +319,13 @@ def file_size_operator(self, src, *args):
             return multiplier * base
         return base
 
+    def basename_operator(self, src, *args):
+        retval = j.BasenameOperator(src)
+        if len(args) > 0:
+            retval = retval.replace(args[0], "")
+
+        return retval
+
     def translate_apply(
         self, expr: WDL.Expr.Apply, **expr_kwargs
     ) -> Union[j.Selector, List[j.Selector]]:
@@ -331,7 +342,7 @@ def translate_apply(
             "_land": j.AndOperator,
             "defined": j.IsDefined,
             "select_first": j.FilterNullOperator,
-            "basename": j.BasenameOperator,
+            "basename": self.basename_operator,
             "length": j.LengthOperator,
             "_gt": j.GtOperator,
             "_gte": j.GteOperator,
@@ -351,6 +362,8 @@ def translate_apply(
             "write_lines": lambda exp: f"JANIS: write_lines({exp})",
             "size": self.file_size_operator,
             "ceil": j.CeilOperator,
+            "select_all": j.FilterNullOperator,
+            "sub": j.ReplaceOperator
         }
         fn = fn_map.get(expr.function_name)
         if fn is None:
@@ -396,7 +409,8 @@ def parse_command_tool_output(self, outp: WDL.Decl):
 
 
 if __name__ == "__main__":
-    doc = "path/to/doc.wdl"
+    # doc = "path/to/doc.wdl"
+    doc = "/Users/michael.franklin/source/wdlz/cramqc.wdl"
     t = WdlParser.from_doc(doc)
-
-    t.translate("janis")
+    t.get_dot_plot(log_to_stdout=True)
+    t.translate("hail")

janis_core/operators/selectors.py

@@ -245,6 +245,10 @@ def basename(self):
 
         return BasenameOperator(self)
 
+    def replace(self, pattern, replacement):
+        from .standard import ReplaceOperator
+        return ReplaceOperator(self, pattern, replacement)
+
     def file_size(self):
         from .standard import FileSizeOperator
 

janis_core/operators/standard.py

@@ -495,6 +495,36 @@ def evaluate(self, inputs):
         return [i for i in iterable if i is not None]
 
 
+class ReplaceOperator(Operator):
+
+    @staticmethod
+    def friendly_signature():
+        return "Base: String, Pattern: String, Replacement: String -> String"
+
+    def argtypes(self) -> List[DataType]:
+        return [String(), String(), String()]
+
+    def evaluate(self, inputs):
+        base, pattern, replacement = [self.evaluate_arg(a, inputs) for a in self.args]
+        import re
+        return re.sub(pattern, replacement, base)
+
+    def to_wdl(self, unwrap_operator, *args):
+        base, pattern, replacement = [unwrap_operator(a) for a in self.args]
+        return f"sub({base}, {pattern}, {replacement})"
+
+    def to_cwl(self, unwrap_operator, *args):
+        base, pattern, replacement = [unwrap_operator(a) for a in self.args]
+        return f"{base}.replace(new RegExp({pattern}), {replacement})"
+
+    def to_python(self, unwrap_operator, *args):
+        base, pattern, replacement = [unwrap_operator(a) for a in self.args]
+        return f"re.sub({pattern}, {replacement}, {base})"
+
+    def returntype(self) -> DataType:
+        return String()
+
+
 # class Stdout(Operator):
 #     @staticmethod
 #     def friendly_signature():

janis_core/operators/stringformatter.py

@@ -66,20 +66,10 @@ def to_wdl(self, unwrap_operator, *args):
         raise Exception("Don't use this method")
 
     def to_python(self, unwrap_operator, *args):
-        # TO avoid format errors when unwrapping StringFormatter, we'll use the following dictionary
-        # that returns the key if it's missing:
-        class DefaultDictionary(dict):
-            def __missing__(self, key):
-                return "{{" + str(key) + "}}"
-
-        d = DefaultDictionary(
-            {
-                # keep in curly braces for the
-                str(k): f"{{{unwrap_operator(v)}}}"
-                for k, v in self.kwargs.items()
-            }
-        )
-        return self._format.format_map(d)
+        f = self._format
+        for k, v in self.kwargs.items():
+            f = f.replace(f"{{{str(k)}}}", unwrap_operator(v))
+        return f
 
     def evaluate(self, inputs):
         resolvedvalues = {

janis_core/tests/test_conditionals.py

@@ -44,7 +44,7 @@ def test_switch(self):
 
         w.output("out", source=w.echoswitch)
 
-        _, wdl_tools = WdlTranslator.translate_workflow(w)
+        _, wdl_tools = WdlTranslator.translate_workflow_internal(w)
         expected = """\
 version development
 

janis_core/tests/test_test_runner.py

@@ -2,6 +2,8 @@
 import os
 from typing import Optional, List, Dict, Union
 from unittest import TestCase, mock
+from unittest.case import skipUnless
+
 from janis_core.tool.test_suite_runner import ToolTestSuiteRunner
 from janis_core.tool.test_classes import (
     TTestCase,
@@ -15,6 +17,13 @@
 from janis_core.tool.test_suite_runner import ToolTestSuiteRunner
 from nose.tools import nottest
 
+has_janis_assistant = False
+try:
+    import janis_assistant
+    has_janis_assistant = True
+except ImportError:
+    pass
+
 
 valid_url = "https://abc.com/some_dir/expected_output_file.txt"
 valid_url_2 = "https://abc.com/some_dir/diff_file.txt"
@@ -165,6 +174,7 @@ def test_get_value_to_compare(self):
         assert runner.get_value_to_compare(t2, file_path) == 3
 
     @nottest
+    @skipUnless(has_janis_assistant, reason="Janis assistant is required to test downloading remote files")
     @mock.patch("urllib.request.urlopen", side_effect=mocked_urllib_urlopen)
     @mock.patch("urllib.request.urlretrieve", side_effect=mocked_urllib_urlretrieve)
     def test_download_remote_files(self, mock_urlopen, mock_urlretrieve):

janis_core/tests/test_translation_cwl.py

@@ -440,14 +440,14 @@ def test_string_formatter_two_param(self):
 
     def test_escaped_characters(self):
         trans = cwl.CwlTranslator
-        translated = trans.translate_tool_internal(TestTool())
+        translated = trans.translate_command_tool_internal(TestTool())
         arg: cwlgen.CommandLineBinding = translated.arguments[0]
         self.assertEqual('test:\\\\t:escaped:\\\\n:characters\\"', arg.valueFrom)
 
 
 class TestCwlEnvVar(unittest.TestCase):
     def test_environment1(self):
-        t = CwlTranslator().translate_tool_internal(tool=TestTool())
+        t = CwlTranslator().translate_command_tool_internal(tool=TestTool())
         envvar: cwlgen.EnvVarRequirement = [
             t for t in t.requirements if t.class_ == "EnvVarRequirement"
         ][0]
@@ -597,11 +597,11 @@ class TestEmptyContainer(unittest.TestCase):
     def test_empty_container_raises(self):
 
         self.assertRaises(
-            Exception, CwlTranslator().translate_tool_internal, SingleTestTool()
+            Exception, CwlTranslator().translate_command_tool_internal, SingleTestTool()
         )
 
     def test_empty_container(self):
-        c = CwlTranslator().translate_tool_internal(
+        c = CwlTranslator().translate_command_tool_internal(
             SingleTestTool(), allow_empty_container=True
         )
         self.assertNotIn("DockerRequirement", c.requirements)
@@ -613,9 +613,10 @@ def test_add_single_to_array_edge(self):
         w.input("inp1", str)
         w.step("stp1", ArrayTestTool(inps=w.inp1))
 
-        c, _, _ = CwlTranslator().translate(
-            w, to_console=False, allow_empty_container=True
+        c, _, _ = CwlTranslator().translate_workflow(
+            w, allow_empty_container=True, to_console=False
         )
+        self.maxDiff = None
         self.assertEqual(cwl_multiinput, c)
 
 
@@ -793,7 +794,7 @@ def test_string_formatter(self):
         )
         wf.step("print", EchoTestTool(inp=wf.readGroupHeaderLine))
         wf.output("out", source=wf.print)
-        d, _ = cwl.CwlTranslator.translate_workflow(
+        d, _ = cwl.CwlTranslator.translate_workflow_internal(
             wf, with_container=False, allow_empty_container=True
         )
         stepinputs = d.save()["steps"][0]["in"]
@@ -822,7 +823,7 @@ def test_string_formatter_stepinput(self):
             ),
         )
         wf.output("out", source=wf.print)
-        d, _ = cwl.CwlTranslator.translate_workflow(
+        d, _ = cwl.CwlTranslator.translate_workflow_internal(
             wf, with_container=False, allow_empty_container=True
         )
         stepinputs = d.save()["steps"][0]["in"]
@@ -862,7 +863,7 @@ def test_two_similar_tools(self):
         w.step("stp1", TestTool(testtool=w.inp))
         w.step("stp2", TestToolV2(testtool=w.inp))
 
-        wf_cwl, _ = CwlTranslator.translate_workflow(w)
+        wf_cwl, _ = CwlTranslator.translate_workflow_internal(w)
         stps = {stp.id: stp for stp in wf_cwl.steps}
 
         self.assertEqual("tools/TestTranslationtool.cwl", stps["stp1"].run)
@@ -871,7 +872,7 @@ def test_two_similar_tools(self):
 
 class TestCwlResourceOperators(unittest.TestCase):
     def test_1(self):
-        tool_cwl = CwlTranslator.translate_tool_internal(
+        tool_cwl = CwlTranslator.translate_command_tool_internal(
             OperatorResourcesTestTool(), with_resource_overrides=True
         )
         resourcereq = [
@@ -899,7 +900,7 @@ def test_read_contents_string(self):
             version="-1",
         )
 
-        translated = CwlTranslator.translate_tool_internal(
+        translated = CwlTranslator.translate_command_tool_internal(
             t, allow_empty_container=True
         )
         self.assertTrue(translated.outputs[0].outputBinding.loadContents)
@@ -914,7 +915,7 @@ def test_read_contents_as_int(self):
             container=None,
             version="-1",
         )
-        translated = CwlTranslator.translate_tool_internal(
+        translated = CwlTranslator.translate_command_tool_internal(
             t, allow_empty_container=True
         )
         self.assertTrue(translated.outputs[0].outputBinding.loadContents)
@@ -962,7 +963,7 @@ def test_filter_null(self):
         w.step("stp", T(inp=FilterNullOperator(w.inp)), scatter="inp")
         w.output("out", source=w.stp.out)
 
-        w_cwl = cwl.CwlTranslator().translate_workflow(w, with_container=False)[0]
+        w_cwl = cwl.CwlTranslator().translate_workflow_internal(w, with_container=False)[0]
         self.assertEqual(2, len(w_cwl.steps))
         self.assertEqual(
             "_evaluate_prescatter-stp-inp/out", w_cwl.steps[1].in_[0].source
@@ -976,7 +977,7 @@ def test_read_contents(self):
         w.step("stp", EchoTestTool(inp=w.inp))
         w.output("out", source=w.stp.out.contents())
 
-        w_cwl = cwl.CwlTranslator().translate_workflow(w, with_container=False)[0]
+        w_cwl = cwl.CwlTranslator().translate_workflow_internal(w, with_container=False)[0]
 
         self.assertEqual(2, len(w_cwl.steps))
         self.assertEqual(
@@ -1025,7 +1026,8 @@ class TestForEachSelectors(unittest.TestCase):
     def test_minimal(self):
         tool = TestForEach()
         # tool.translate("cwl", export_path="~/Desktop/tmp", to_disk=True)
-        w, _ = CwlTranslator.translate_workflow(tool)
+        w, _ = CwlTranslator.translate_workflow_internal(tool)
+        tool.translate("cwl")
 
         stp = w.steps[0]
         self.assertEqual("inp", stp.in_[0].source)

janis_core/tests/test_translation_wdl.py

@@ -389,7 +389,7 @@ def test_string_formatter_two_param(self):
 
     def test_escaped_characters(self):
         trans = wdl.WdlTranslator
-        translated = trans.translate_tool_internal(TestTool())
+        translated = trans.translate_command_tool_internal(TestTool())
         arg = translated.command[-1].arguments[0]
         self.assertEqual("'test:\\t:escaped:\\n:characters\"'", arg.value)
 
@@ -722,7 +722,7 @@ def test_string_optional_default(self):
 
 class TestWdlEnvVar(unittest.TestCase):
     def test_environment1(self):
-        t = WdlTranslator().translate_tool_internal(tool=TestTool())
+        t = WdlTranslator().translate_command_tool_internal(tool=TestTool())
         s = t.get_string()
         print(s)
 
@@ -1095,7 +1095,7 @@ def test_two_similar_tools(self):
         w.step("stp1", TestTool(testtool=w.inp))
         w.step("stp2", TestToolV2(testtool=w.inp))
 
-        wf_wdl, _ = WdlTranslator.translate_workflow(w)
+        wf_wdl, _ = WdlTranslator.translate_workflow_internal(w)
 
         expected = """\
 version development
@@ -1177,7 +1177,7 @@ def test_array_secondary_connection(self):
 
     def test_workflow_secondary_outputs(self):
         wf = TestWorkflowThatOutputsArraysOfSecondaryFiles()
-        wfwdl, _ = WdlTranslator.translate_workflow(wf)
+        wfwdl, _ = WdlTranslator.translate_workflow_internal(wf)
 
         outs = [o.get_string() for o in wfwdl.outputs]
         self.assertEqual("Array[File] out = stp.out", outs[0])
@@ -1449,7 +1449,7 @@ def test_with_str_default(self):
 
 class TestWdlResourceOperators(unittest.TestCase):
     def test_1(self):
-        tool_wdl = WdlTranslator.translate_tool_internal(
+        tool_wdl = WdlTranslator.translate_command_tool_internal(
             OperatorResourcesTestTool(), with_resource_overrides=True
         ).get_string()
         lines = tool_wdl.splitlines(keepends=False)
@@ -1465,7 +1465,7 @@ def test_1(self):
         self.assertEqual("duration: select_first([runtime_seconds, 60, 86400])", time)
 
     def test_base(self):
-        tool_wdl = WdlTranslator.translate_tool_internal(
+        tool_wdl = WdlTranslator.translate_command_tool_internal(
             EchoTestTool(), with_resource_overrides=True
         ).get_string()
         lines = tool_wdl.splitlines(keepends=False)
@@ -1601,7 +1601,7 @@ def test_file_int_fail(self):
 class TestForEachSelectors(unittest.TestCase):
     def test_minimal(self):
         TestForEach().translate("wdl", to_disk=True, export_path="~/Desktop/tmp")
-        w, _ = WdlTranslator.translate_workflow(TestForEach())
+        w, _ = WdlTranslator.translate_workflow_internal(TestForEach())
         expected = """\
 version development
 
@@ -1636,3 +1636,4 @@ def test_minimal(self):
     container=None,
     version="-1",
 )
+

janis_core/tests/testtools.py

@@ -19,6 +19,7 @@
     InputQualityType,
     Workflow,
     ForEachSelector,
+    RangeOperator
 )
 
 
@@ -328,3 +329,17 @@ def friendly_name(self):
 
     def id(self) -> str:
         return "TestForEach"
+
+class TestForEachWithOperation(Workflow):
+    def constructor(self):
+        self.input("inp", Array(str))
+        self.step(
+            "print", EchoTestTool(inp=self.inp[ForEachSelector()] + ForEachSelector() + "-hello"), _foreach=RangeOperator(self.inp.length())
+        )
+        self.output("out", source=self.print.out)
+
+    def friendly_name(self):
+        return self.id()
+
+    def id(self) -> str:
+        return "TestForEach"