Add Hail batch + translator base refactor

janis_core/deps/__init__.py

@@ -1,2 +1,4 @@
-import cwl_utils.parser_v1_2 as cwlgen
+from cwl_utils import parser
+
+cwlgen = parser.cwl_v1_2
 import wdlgen

janis_core/ingestion/fromwdl.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 import functools
 import os
 import re
@@ -194,7 +195,7 @@ def parse_memory_requirement(self, value):
             raise Exception(f"Memory type {s}")
         elif isinstance(s, (float, int)):
             # in bytes?
-            return s / (1024 ** 3)
+            return s / (1024**3)
         elif isinstance(s, j.Selector):
             return s
         raise Exception(f"Couldn't recognise memory requirement '{value}'")
@@ -238,7 +239,11 @@ def from_loaded_task(self, obj: WDL.Task):
         inputs = obj.inputs
 
         cpus = self.translate_expr(rt.get("cpu"))
-        if not isinstance(cpus, j.Selector) and cpus is not None and not isinstance(cpus, (int, float)):
+        if (
+            cpus is not None
+            and not isinstance(cpus, j.Selector)
+            and not isinstance(cpus, (int, float))
+        ):
             cpus = int(cpus)
 
         c = j.CommandToolBuilder(
@@ -272,7 +277,10 @@ def translate_expr(
 
         if isinstance(expr, WDL.Expr.Array):
             # a literal array
-            return [self.translate_expr(e) for e in expr.items]
+            return [
+                self.translate_expr(e, input_selector_getter=input_selector_getter)
+                for e in expr.items
+            ]
         if isinstance(expr, WDL.Expr.String):
             return self.translate_wdl_string(expr)
         elif isinstance(expr, (WDL.Expr.Int, WDL.Expr.Boolean, WDL.Expr.Float)):
@@ -285,7 +293,7 @@ def translate_expr(
             n = str(expr.expr)
             if input_selector_getter:
                 return input_selector_getter(n)
-            return j.InputSelector(n)
+            return j.InputSelector(str(n))
         elif isinstance(expr, WDL.Expr.Apply):
             return self.translate_apply(
                 expr, input_selector_getter=input_selector_getter
@@ -394,8 +402,12 @@ def translate_apply(
             "round": j.RoundOperator,
             "write_lines": lambda exp: f"JANIS: write_lines({exp})",
             "read_tsv": lambda exp: f"JANIS: j.read_tsv({exp})",
+            "write_tsv": lambda exp: f"JANIS: j.write_tsv({exp})",
             "read_boolean": lambda exp: f"JANIS: j.read_boolean({exp})",
             "read_lines": lambda exp: f"JANIS: j.read_lines({exp})",
+            "zip": lambda exp: f"JANIS: j.zip({exp})",
+            "transpose": j.TransposeOperator,
+            "read_string": j.ReadContents,
         }
         fn = fn_map.get(expr.function_name)
         if fn is None:
@@ -420,6 +432,8 @@ def parse_wdl_type(self, t: WDL.Type.Base):
             return j.Directory(optional=optional)
         elif isinstance(t, WDL.Type.Array):
             return j.Array(self.parse_wdl_type(t.item_type), optional=optional)
+        elif isinstance(t, WDL.Type.Pair):
+            return j.Array(self.parse_wdl_type(t.left_type), optional=optional)
 
         raise Exception(f"Didn't handle WDL type conversion for '{t}' ({type(t)})")
 

janis_core/operators/standard.py

@@ -25,6 +25,7 @@ def argtypes(self) -> List[DataType]:
         return [File()]
 
     def to_python(self, unwrap_operator, *args):
+        return f"open({unwrap_operator(args[0])!r}).read()"
         raise NotImplementedError("Determine _safe_ one line solution for ReadContents")
 
     def to_wdl(self, unwrap_operator, *args):
@@ -472,7 +473,7 @@ def returntype(self):
                 if not isinstance(self.args[0], InputSelector):
                     Logger.warn(
                         f'Expected return type of "{self.args[0]}" to be an array, '
-                        f'but found {outer_rettype}, will return this as a returntype.'
+                        f"but found {outer_rettype}, will return this as a returntype."
                     )
             else:
                 rettype = outer_rettype.subtype()
@@ -506,7 +507,6 @@ def evaluate(self, inputs):
 
 
 class ReplaceOperator(Operator):
-
     @staticmethod
     def friendly_signature():
         return "Base: String, Pattern: String, Replacement: String -> String"
@@ -517,6 +517,7 @@ def argtypes(self) -> List[DataType]:
     def evaluate(self, inputs):
         base, pattern, replacement = [self.evaluate_arg(a, inputs) for a in self.args]
         import re
+
         return re.sub(pattern, replacement, base)
 
     def to_wdl(self, unwrap_operator, *args):

janis_core/tests/test_conditionals.py

@@ -44,7 +44,7 @@ def test_switch(self):
 
         w.output("out", source=w.echoswitch)
 
-        _, wdl_tools = WdlTranslator.translate_workflow(w)
+        _, wdl_tools = WdlTranslator.translate_workflow_internal(w)
         expected = """\
 version development
 

janis_core/tests/test_test_runner.py

@@ -2,6 +2,8 @@
 import os
 from typing import Optional, List, Dict, Union
 from unittest import TestCase, mock
+from unittest.case import skipUnless
+
 from janis_core.tool.test_suite_runner import ToolTestSuiteRunner
 from janis_core.tool.test_classes import (
     TTestCase,
@@ -15,6 +17,13 @@
 from janis_core.tool.test_suite_runner import ToolTestSuiteRunner
 from nose.tools import nottest
 
+has_janis_assistant = False
+try:
+    import janis_assistant
+    has_janis_assistant = True
+except ImportError:
+    pass
+
 
 valid_url = "https://abc.com/some_dir/expected_output_file.txt"
 valid_url_2 = "https://abc.com/some_dir/diff_file.txt"
@@ -165,6 +174,7 @@ def test_get_value_to_compare(self):
         assert runner.get_value_to_compare(t2, file_path) == 3
 
     @nottest
+    @skipUnless(has_janis_assistant, reason="Janis assistant is required to test downloading remote files")
     @mock.patch("urllib.request.urlopen", side_effect=mocked_urllib_urlopen)
     @mock.patch("urllib.request.urlretrieve", side_effect=mocked_urllib_urlretrieve)
     def test_download_remote_files(self, mock_urlopen, mock_urlretrieve):

janis_core/tests/test_translation_cwl.py

@@ -440,14 +440,14 @@ def test_string_formatter_two_param(self):
 
     def test_escaped_characters(self):
         trans = cwl.CwlTranslator
-        translated = trans.translate_tool_internal(TestTool())
+        translated = trans.translate_command_tool_internal(TestTool())
         arg: cwlgen.CommandLineBinding = translated.arguments[0]
         self.assertEqual('test:\\\\t:escaped:\\\\n:characters\\"', arg.valueFrom)
 
 
 class TestCwlEnvVar(unittest.TestCase):
     def test_environment1(self):
-        t = CwlTranslator().translate_tool_internal(tool=TestTool())
+        t = CwlTranslator().translate_command_tool_internal(tool=TestTool())
         envvar: cwlgen.EnvVarRequirement = [
             t for t in t.requirements if t.class_ == "EnvVarRequirement"
         ][0]
@@ -597,11 +597,11 @@ class TestEmptyContainer(unittest.TestCase):
     def test_empty_container_raises(self):
 
         self.assertRaises(
-            Exception, CwlTranslator().translate_tool_internal, SingleTestTool()
+            Exception, CwlTranslator().translate_command_tool_internal, SingleTestTool()
         )
 
     def test_empty_container(self):
-        c = CwlTranslator().translate_tool_internal(
+        c = CwlTranslator().translate_command_tool_internal(
             SingleTestTool(), allow_empty_container=True
         )
         self.assertNotIn("DockerRequirement", c.requirements)
@@ -613,9 +613,10 @@ def test_add_single_to_array_edge(self):
         w.input("inp1", str)
         w.step("stp1", ArrayTestTool(inps=w.inp1))
 
-        c, _, _ = CwlTranslator().translate(
-            w, to_console=False, allow_empty_container=True
+        c, _, _ = CwlTranslator().translate_workflow(
+            w, allow_empty_container=True, to_console=False
         )
+        self.maxDiff = None
         self.assertEqual(cwl_multiinput, c)
 
 
@@ -794,7 +795,7 @@ def test_string_formatter(self):
         )
         wf.step("print", EchoTestTool(inp=wf.readGroupHeaderLine))
         wf.output("out", source=wf.print)
-        d, _ = cwl.CwlTranslator.translate_workflow(
+        d, _ = cwl.CwlTranslator.translate_workflow_internal(
             wf, with_container=False, allow_empty_container=True
         )
         stepinputs = d.save()["steps"][0]["in"]
@@ -823,7 +824,7 @@ def test_string_formatter_stepinput(self):
             ),
         )
         wf.output("out", source=wf.print)
-        d, _ = cwl.CwlTranslator.translate_workflow(
+        d, _ = cwl.CwlTranslator.translate_workflow_internal(
             wf, with_container=False, allow_empty_container=True
         )
         stepinputs = d.save()["steps"][0]["in"]
@@ -863,7 +864,7 @@ def test_two_similar_tools(self):
         w.step("stp1", TestTool(testtool=w.inp))
         w.step("stp2", TestToolV2(testtool=w.inp))
 
-        wf_cwl, _ = CwlTranslator.translate_workflow(w)
+        wf_cwl, _ = CwlTranslator.translate_workflow_internal(w)
         stps = {stp.id: stp for stp in wf_cwl.steps}
 
         self.assertEqual("tools/TestTranslationtool.cwl", stps["stp1"].run)
@@ -872,7 +873,7 @@ def test_two_similar_tools(self):
 
 class TestCwlResourceOperators(unittest.TestCase):
     def test_1(self):
-        tool_cwl = CwlTranslator.translate_tool_internal(
+        tool_cwl = CwlTranslator.translate_command_tool_internal(
             OperatorResourcesTestTool(), with_resource_overrides=True
         )
         resourcereq = [
@@ -900,7 +901,7 @@ def test_read_contents_string(self):
             version="-1",
         )
 
-        translated = CwlTranslator.translate_tool_internal(
+        translated = CwlTranslator.translate_command_tool_internal(
             t, allow_empty_container=True
         )
         self.assertTrue(translated.outputs[0].outputBinding.loadContents)
@@ -915,7 +916,7 @@ def test_read_contents_as_int(self):
             container=None,
             version="-1",
         )
-        translated = CwlTranslator.translate_tool_internal(
+        translated = CwlTranslator.translate_command_tool_internal(
             t, allow_empty_container=True
         )
         self.assertTrue(translated.outputs[0].outputBinding.loadContents)
@@ -963,7 +964,7 @@ def test_filter_null(self):
         w.step("stp", T(inp=FilterNullOperator(w.inp)), scatter="inp")
         w.output("out", source=w.stp.out)
 
-        w_cwl = cwl.CwlTranslator().translate_workflow(w, with_container=False)[0]
+        w_cwl = cwl.CwlTranslator().translate_workflow_internal(w, with_container=False)[0]
         self.assertEqual(2, len(w_cwl.steps))
         self.assertEqual(
             "_evaluate_prescatter-stp-inp/out", w_cwl.steps[1].in_[0].source
@@ -977,7 +978,7 @@ def test_read_contents(self):
         w.step("stp", EchoTestTool(inp=w.inp))
         w.output("out", source=w.stp.out.contents())
 
-        w_cwl = cwl.CwlTranslator().translate_workflow(w, with_container=False)[0]
+        w_cwl = cwl.CwlTranslator().translate_workflow_internal(w, with_container=False)[0]
 
         self.assertEqual(2, len(w_cwl.steps))
         self.assertEqual(
@@ -1026,7 +1027,8 @@ class TestForEachSelectors(unittest.TestCase):
     def test_minimal(self):
         tool = TestForEach()
         # tool.translate("cwl", export_path="~/Desktop/tmp", to_disk=True)
-        w, _ = CwlTranslator.translate_workflow(tool)
+        w, _ = CwlTranslator.translate_workflow_internal(tool)
+        tool.translate("cwl")
 
         stp = w.steps[0]
         self.assertEqual("inp", stp.in_[0].source)

janis_core/tests/test_translation_wdl.py

@@ -390,7 +390,7 @@ def test_string_formatter_two_param(self):
 
     def test_escaped_characters(self):
         trans = wdl.WdlTranslator
-        translated = trans.translate_tool_internal(TestTool())
+        translated = trans.translate_command_tool_internal(TestTool())
         arg = translated.command[-1].arguments[0]
         self.assertEqual("'test:\\t:escaped:\\n:characters\"'", arg.value)
 
@@ -723,7 +723,7 @@ def test_string_optional_default(self):
 
 class TestWdlEnvVar(unittest.TestCase):
     def test_environment1(self):
-        t = WdlTranslator().translate_tool_internal(tool=TestTool())
+        t = WdlTranslator().translate_command_tool_internal(tool=TestTool())
         s = t.get_string()
         print(s)
 
@@ -1096,7 +1096,7 @@ def test_two_similar_tools(self):
         w.step("stp1", TestTool(testtool=w.inp))
         w.step("stp2", TestToolV2(testtool=w.inp))
 
-        wf_wdl, _ = WdlTranslator.translate_workflow(w)
+        wf_wdl, _ = WdlTranslator.translate_workflow_internal(w)
 
         expected = """\
 version development
@@ -1178,7 +1178,7 @@ def test_array_secondary_connection(self):
 
     def test_workflow_secondary_outputs(self):
         wf = TestWorkflowThatOutputsArraysOfSecondaryFiles()
-        wfwdl, _ = WdlTranslator.translate_workflow(wf)
+        wfwdl, _ = WdlTranslator.translate_workflow_internal(wf)
 
         outs = [o.get_string() for o in wfwdl.outputs]
         self.assertEqual("Array[File] out = stp.out", outs[0])
@@ -1450,7 +1450,7 @@ def test_with_str_default(self):
 
 class TestWdlResourceOperators(unittest.TestCase):
     def test_1(self):
-        tool_wdl = WdlTranslator.translate_tool_internal(
+        tool_wdl = WdlTranslator.translate_command_tool_internal(
             OperatorResourcesTestTool(), with_resource_overrides=True
         ).get_string()
         lines = tool_wdl.splitlines(keepends=False)
@@ -1466,7 +1466,7 @@ def test_1(self):
         self.assertEqual("duration: select_first([runtime_seconds, 60, 86400])", time)
 
     def test_base(self):
-        tool_wdl = WdlTranslator.translate_tool_internal(
+        tool_wdl = WdlTranslator.translate_command_tool_internal(
             EchoTestTool(), with_resource_overrides=True
         ).get_string()
         lines = tool_wdl.splitlines(keepends=False)

janis_core/tests/testtools.py

@@ -19,6 +19,7 @@
     InputQualityType,
     Workflow,
     ForEachSelector,
+    RangeOperator
 )
 
 
@@ -328,3 +329,17 @@ def friendly_name(self):
 
     def id(self) -> str:
         return "TestForEach"
+
+class TestForEachWithOperation(Workflow):
+    def constructor(self):
+        self.input("inp", Array(str))
+        self.step(
+            "print", EchoTestTool(inp=self.inp[ForEachSelector()] + ForEachSelector() + "-hello"), _foreach=RangeOperator(self.inp.length())
+        )
+        self.output("out", source=self.print.out)
+
+    def friendly_name(self):
+        return self.id()
+
+    def id(self) -> str:
+        return "TestForEach"

janis_core/translationdeps/supportedtranslations.py

@@ -5,6 +5,7 @@ class SupportedTranslation(Enum):
     CWL = "cwl"
     WDL = "wdl"
     JANIS = "janis"
+    HAILBATCH = "hail"
 
     def __str__(self):
         return self.value
@@ -27,10 +28,15 @@ def get_translator(self):
 
             return JanisTranslator()
 
+        elif self == SupportedTranslation.HAILBATCH:
+            from ..translations.hailbatch import HailBatchTranslator
+
+            return HailBatchTranslator()
+
     @staticmethod
     def all():
         return [
             SupportedTranslation.CWL,
             SupportedTranslation.WDL,
-            SupportedTranslation.JANIS,
+            SupportedTranslation.JANIS
         ]