Add Hail batch + translator base refactor

janis_core/ingestion/fromwdl.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 import functools
 import os
 import re
@@ -390,7 +391,6 @@ def translate_apply(
             "read_tsv": lambda exp: f'JANIS: j.read_tsv({exp})',
             "read_boolean": lambda exp: f'JANIS: j.read_boolean({exp})',
             'read_lines': lambda exp: f'JANIS: j.read_lines({exp})',
-
         }
         fn = fn_map.get(expr.function_name)
         if fn is None:
@@ -436,6 +436,12 @@ def parse_command_tool_output(self, outp: WDL.Decl):
 
 
 if __name__ == "__main__":
+    # doc = "path/to/doc.wdl"
+    doc = "/Users/michael.franklin/source/wdlz/cramqc.wdl"
+    t = WdlParser.from_doc(doc)
+    t.get_dot_plot(log_to_stdout=True)
+    t.translate("hail")
+
     import sys
     if len(sys.argv) != 2:
         raise Exception("Expected 1 argument, the name of a CWL tool.")

janis_core/tests/test_conditionals.py

@@ -44,7 +44,7 @@ def test_switch(self):
 
         w.output("out", source=w.echoswitch)
 
-        _, wdl_tools = WdlTranslator.translate_workflow(w)
+        _, wdl_tools = WdlTranslator.translate_workflow_internal(w)
         expected = """\
 version development
 

janis_core/tests/test_test_runner.py

@@ -2,6 +2,8 @@
 import os
 from typing import Optional, List, Dict, Union
 from unittest import TestCase, mock
+from unittest.case import skipUnless
+
 from janis_core.tool.test_suite_runner import ToolTestSuiteRunner
 from janis_core.tool.test_classes import (
     TTestCase,
@@ -15,6 +17,13 @@
 from janis_core.tool.test_suite_runner import ToolTestSuiteRunner
 from nose.tools import nottest
 
+has_janis_assistant = False
+try:
+    import janis_assistant
+    has_janis_assistant = True
+except ImportError:
+    pass
+
 
 valid_url = "https://abc.com/some_dir/expected_output_file.txt"
 valid_url_2 = "https://abc.com/some_dir/diff_file.txt"
@@ -165,6 +174,7 @@ def test_get_value_to_compare(self):
         assert runner.get_value_to_compare(t2, file_path) == 3
 
     @nottest
+    @skipUnless(has_janis_assistant, reason="Janis assistant is required to test downloading remote files")
     @mock.patch("urllib.request.urlopen", side_effect=mocked_urllib_urlopen)
     @mock.patch("urllib.request.urlretrieve", side_effect=mocked_urllib_urlretrieve)
     def test_download_remote_files(self, mock_urlopen, mock_urlretrieve):

janis_core/tests/test_translation_cwl.py

@@ -440,14 +440,14 @@ def test_string_formatter_two_param(self):
 
     def test_escaped_characters(self):
         trans = cwl.CwlTranslator
-        translated = trans.translate_tool_internal(TestTool())
+        translated = trans.translate_command_tool_internal(TestTool())
         arg: cwlgen.CommandLineBinding = translated.arguments[0]
         self.assertEqual('test:\\\\t:escaped:\\\\n:characters\\"', arg.valueFrom)
 
 
 class TestCwlEnvVar(unittest.TestCase):
     def test_environment1(self):
-        t = CwlTranslator().translate_tool_internal(tool=TestTool())
+        t = CwlTranslator().translate_command_tool_internal(tool=TestTool())
         envvar: cwlgen.EnvVarRequirement = [
             t for t in t.requirements if t.class_ == "EnvVarRequirement"
         ][0]
@@ -597,11 +597,11 @@ class TestEmptyContainer(unittest.TestCase):
     def test_empty_container_raises(self):
 
         self.assertRaises(
-            Exception, CwlTranslator().translate_tool_internal, SingleTestTool()
+            Exception, CwlTranslator().translate_command_tool_internal, SingleTestTool()
         )
 
     def test_empty_container(self):
-        c = CwlTranslator().translate_tool_internal(
+        c = CwlTranslator().translate_command_tool_internal(
             SingleTestTool(), allow_empty_container=True
         )
         self.assertNotIn("DockerRequirement", c.requirements)
@@ -613,9 +613,10 @@ def test_add_single_to_array_edge(self):
         w.input("inp1", str)
         w.step("stp1", ArrayTestTool(inps=w.inp1))
 
-        c, _, _ = CwlTranslator().translate(
-            w, to_console=False, allow_empty_container=True
+        c, _, _ = CwlTranslator().translate_workflow(
+            w, allow_empty_container=True, to_console=False
         )
+        self.maxDiff = None
         self.assertEqual(cwl_multiinput, c)
 
 
@@ -794,7 +795,7 @@ def test_string_formatter(self):
         )
         wf.step("print", EchoTestTool(inp=wf.readGroupHeaderLine))
         wf.output("out", source=wf.print)
-        d, _ = cwl.CwlTranslator.translate_workflow(
+        d, _ = cwl.CwlTranslator.translate_workflow_internal(
             wf, with_container=False, allow_empty_container=True
         )
         stepinputs = d.save()["steps"][0]["in"]
@@ -823,7 +824,7 @@ def test_string_formatter_stepinput(self):
             ),
         )
         wf.output("out", source=wf.print)
-        d, _ = cwl.CwlTranslator.translate_workflow(
+        d, _ = cwl.CwlTranslator.translate_workflow_internal(
             wf, with_container=False, allow_empty_container=True
         )
         stepinputs = d.save()["steps"][0]["in"]
@@ -863,7 +864,7 @@ def test_two_similar_tools(self):
         w.step("stp1", TestTool(testtool=w.inp))
         w.step("stp2", TestToolV2(testtool=w.inp))
 
-        wf_cwl, _ = CwlTranslator.translate_workflow(w)
+        wf_cwl, _ = CwlTranslator.translate_workflow_internal(w)
         stps = {stp.id: stp for stp in wf_cwl.steps}
 
         self.assertEqual("tools/TestTranslationtool.cwl", stps["stp1"].run)
@@ -872,7 +873,7 @@ def test_two_similar_tools(self):
 
 class TestCwlResourceOperators(unittest.TestCase):
     def test_1(self):
-        tool_cwl = CwlTranslator.translate_tool_internal(
+        tool_cwl = CwlTranslator.translate_command_tool_internal(
             OperatorResourcesTestTool(), with_resource_overrides=True
         )
         resourcereq = [
@@ -900,7 +901,7 @@ def test_read_contents_string(self):
             version="-1",
         )
 
-        translated = CwlTranslator.translate_tool_internal(
+        translated = CwlTranslator.translate_command_tool_internal(
             t, allow_empty_container=True
         )
         self.assertTrue(translated.outputs[0].outputBinding.loadContents)
@@ -915,7 +916,7 @@ def test_read_contents_as_int(self):
             container=None,
             version="-1",
         )
-        translated = CwlTranslator.translate_tool_internal(
+        translated = CwlTranslator.translate_command_tool_internal(
             t, allow_empty_container=True
         )
         self.assertTrue(translated.outputs[0].outputBinding.loadContents)
@@ -963,7 +964,7 @@ def test_filter_null(self):
         w.step("stp", T(inp=FilterNullOperator(w.inp)), scatter="inp")
         w.output("out", source=w.stp.out)
 
-        w_cwl = cwl.CwlTranslator().translate_workflow(w, with_container=False)[0]
+        w_cwl = cwl.CwlTranslator().translate_workflow_internal(w, with_container=False)[0]
         self.assertEqual(2, len(w_cwl.steps))
         self.assertEqual(
             "_evaluate_prescatter-stp-inp/out", w_cwl.steps[1].in_[0].source
@@ -977,7 +978,7 @@ def test_read_contents(self):
         w.step("stp", EchoTestTool(inp=w.inp))
         w.output("out", source=w.stp.out.contents())
 
-        w_cwl = cwl.CwlTranslator().translate_workflow(w, with_container=False)[0]
+        w_cwl = cwl.CwlTranslator().translate_workflow_internal(w, with_container=False)[0]
 
         self.assertEqual(2, len(w_cwl.steps))
         self.assertEqual(
@@ -1026,7 +1027,8 @@ class TestForEachSelectors(unittest.TestCase):
     def test_minimal(self):
         tool = TestForEach()
         # tool.translate("cwl", export_path="~/Desktop/tmp", to_disk=True)
-        w, _ = CwlTranslator.translate_workflow(tool)
+        w, _ = CwlTranslator.translate_workflow_internal(tool)
+        tool.translate("cwl")
 
         stp = w.steps[0]
         self.assertEqual("inp", stp.in_[0].source)

janis_core/tests/test_translation_wdl.py

@@ -389,7 +389,7 @@ def test_string_formatter_two_param(self):
 
     def test_escaped_characters(self):
         trans = wdl.WdlTranslator
-        translated = trans.translate_tool_internal(TestTool())
+        translated = trans.translate_command_tool_internal(TestTool())
         arg = translated.command[-1].arguments[0]
         self.assertEqual("'test:\\t:escaped:\\n:characters\"'", arg.value)
 
@@ -722,7 +722,7 @@ def test_string_optional_default(self):
 
 class TestWdlEnvVar(unittest.TestCase):
     def test_environment1(self):
-        t = WdlTranslator().translate_tool_internal(tool=TestTool())
+        t = WdlTranslator().translate_command_tool_internal(tool=TestTool())
         s = t.get_string()
         print(s)
 
@@ -1095,7 +1095,7 @@ def test_two_similar_tools(self):
         w.step("stp1", TestTool(testtool=w.inp))
         w.step("stp2", TestToolV2(testtool=w.inp))
 
-        wf_wdl, _ = WdlTranslator.translate_workflow(w)
+        wf_wdl, _ = WdlTranslator.translate_workflow_internal(w)
 
         expected = """\
 version development
@@ -1177,7 +1177,7 @@ def test_array_secondary_connection(self):
 
     def test_workflow_secondary_outputs(self):
         wf = TestWorkflowThatOutputsArraysOfSecondaryFiles()
-        wfwdl, _ = WdlTranslator.translate_workflow(wf)
+        wfwdl, _ = WdlTranslator.translate_workflow_internal(wf)
 
         outs = [o.get_string() for o in wfwdl.outputs]
         self.assertEqual("Array[File] out = stp.out", outs[0])
@@ -1449,7 +1449,7 @@ def test_with_str_default(self):
 
 class TestWdlResourceOperators(unittest.TestCase):
     def test_1(self):
-        tool_wdl = WdlTranslator.translate_tool_internal(
+        tool_wdl = WdlTranslator.translate_command_tool_internal(
             OperatorResourcesTestTool(), with_resource_overrides=True
         ).get_string()
         lines = tool_wdl.splitlines(keepends=False)
@@ -1465,7 +1465,7 @@ def test_1(self):
         self.assertEqual("duration: select_first([runtime_seconds, 60, 86400])", time)
 
     def test_base(self):
-        tool_wdl = WdlTranslator.translate_tool_internal(
+        tool_wdl = WdlTranslator.translate_command_tool_internal(
             EchoTestTool(), with_resource_overrides=True
         ).get_string()
         lines = tool_wdl.splitlines(keepends=False)
@@ -1601,7 +1601,7 @@ def test_file_int_fail(self):
 class TestForEachSelectors(unittest.TestCase):
     def test_minimal(self):
         TestForEach().translate("wdl", to_disk=True, export_path="~/Desktop/tmp")
-        w, _ = WdlTranslator.translate_workflow(TestForEach())
+        w, _ = WdlTranslator.translate_workflow_internal(TestForEach())
         expected = """\
 version development
 
@@ -1636,3 +1636,4 @@ def test_minimal(self):
     container=None,
     version="-1",
 )
+

janis_core/tests/testtools.py

@@ -19,6 +19,7 @@
     InputQualityType,
     Workflow,
     ForEachSelector,
+    RangeOperator
 )
 
 
@@ -328,3 +329,17 @@ def friendly_name(self):
 
     def id(self) -> str:
         return "TestForEach"
+
+class TestForEachWithOperation(Workflow):
+    def constructor(self):
+        self.input("inp", Array(str))
+        self.step(
+            "print", EchoTestTool(inp=self.inp[ForEachSelector()] + ForEachSelector() + "-hello"), _foreach=RangeOperator(self.inp.length())
+        )
+        self.output("out", source=self.print.out)
+
+    def friendly_name(self):
+        return self.id()
+
+    def id(self) -> str:
+        return "TestForEach"

janis_core/translationdeps/supportedtranslations.py

@@ -5,6 +5,7 @@ class SupportedTranslation(Enum):
     CWL = "cwl"
     WDL = "wdl"
     JANIS = "janis"
+    HAILBATCH = "hail"
 
     def __str__(self):
         return self.value
@@ -27,10 +28,15 @@ def get_translator(self):
 
             return JanisTranslator()
 
+        elif self == SupportedTranslation.HAILBATCH:
+            from ..translations.hailbatch import HailBatchTranslator
+
+            return HailBatchTranslator()
+
     @staticmethod
     def all():
         return [
             SupportedTranslation.CWL,
             SupportedTranslation.WDL,
-            SupportedTranslation.JANIS,
+            SupportedTranslation.JANIS
         ]

janis_core/translations/__init__.py

@@ -30,7 +30,6 @@ def translate_workflow(
     should_zip=True,
     merge_resources=False,
     hints=None,
-    allow_null_if_not_optional=True,
     additional_inputs: Dict = None,
     max_cores=None,
     max_mem=None,
@@ -39,7 +38,7 @@ def translate_workflow(
     container_override: dict = None,
 ):
     translator = get_translator(translation)
-    return translator.translate(
+    return translator.translate_workflow(
         workflow,
         to_console=to_console,
         tool_to_console=tool_to_console,
@@ -52,7 +51,6 @@ def translate_workflow(
         should_zip=should_zip,
         merge_resources=merge_resources,
         hints=hints,
-        allow_null_if_not_optional=allow_null_if_not_optional,
         additional_inputs=additional_inputs,
         max_cores=max_cores,
         max_mem=max_mem,