Simplify WDL Toil job graphs

requirements-wdl.txt

@@ -1,2 +1,3 @@
 miniwdl==1.10.0
 wdlparse==0.1.0
+graphlib-backport==1.0 ; python_version < '3.9'

src/toil/common.py

@@ -239,7 +239,7 @@ def prepare_restart(self) -> None:
         # auto-generated and point to a temp directory that could no longer
         # exist and that can't safely be re-made.
         self.write_messages = None
-        
+
 
     def setOptions(self, options: Namespace) -> None:
         """Creates a config object from the options object."""

src/toil/jobStores/aws/jobStore.py

@@ -757,7 +757,7 @@ def bucket_retry_predicate(error):
                                 bucket_tagging.put(Tagging={'TagSet': flat_tags})
 
                             # Configure bucket so that we can make objects in
-                            # it public, which was the historical default. 
+                            # it public, which was the historical default.
                             enable_public_objects(bucket_name)
                         elif block:
                             raise

src/toil/test/__init__.py

@@ -578,7 +578,7 @@ def needs_docker(test_item: MT) -> MT:
         return test_item
     else:
         return unittest.skip("Install docker to include this test.")(test_item)
-        
+
 def needs_singularity(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if

src/toil/test/wdl/wdltoil_test.py

@@ -9,7 +9,7 @@
 
 import pytest
 
-from toil.test import ToilTest, needs_docker, needs_docker_cuda, needs_google_storage, needs_java, needs_singularity_or_docker, slow
+from toil.test import ToilTest, needs_docker_cuda, needs_google_storage, needs_java, needs_singularity_or_docker, slow
 from toil.version import exactPython
 # Don't import the test case directly or pytest will test it again.
 import toil.test.wdl.toilwdlTest
@@ -79,16 +79,16 @@ def setUpClass(cls) -> None:
         """Runs once for all tests."""
         cls.base_command = [exactPython, '-m', 'toil.wdl.wdltoil']
 
-    # We inherit a testMD5sum but it is going to need Singularity and not
-    # Docker now. And also needs to have a WDL 1.0+ WDL file. So we replace it.
+    # We inherit a testMD5sum but it is going to need Singularity or Docker
+    # now. And also needs to have a WDL 1.0+ WDL file. So we replace it.
     @needs_singularity_or_docker
     def testMD5sum(self):
         """Test if Toil produces the same outputs as known good outputs for WDL's
         GATK tutorial #1."""
         wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug'])
+        result_json = subprocess.check_output(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug', '--retryCount=0'])
         result = json.loads(result_json)
 
         assert 'ga4ghMd5.value' in result
@@ -114,7 +114,7 @@ def test_miniwdl_self_test(self):
         wdl_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/self_test.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/inputs.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl'])
+        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '--logDebug', '-o', self.output_dir, '--outputDialect', 'miniwdl'])
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -141,7 +141,7 @@ def test_miniwdl_self_test(self):
     @slow
     @needs_docker_cuda
     def test_giraffe_deepvariant(self):
-        """Test if Giraffe and CPU DeepVariant run. This could take 25 minutes."""
+        """Test if Giraffe and GPU DeepVariant run. This could take 25 minutes."""
         # TODO: enable test if nvidia-container-runtime and Singularity are installed but Docker isn't.
 
         json_dir = self._createTempDir()
@@ -184,7 +184,7 @@ def test_giraffe_deepvariant(self):
     @slow
     @needs_singularity_or_docker
     def test_giraffe(self):
-        """Test if Giraffe runs. This could take 12 minutes. Also we scale it down."""
+        """Test if Giraffe runs. This could take 12 minutes. Also we scale it down but it still demands lots of memory."""
         # TODO: enable test if nvidia-container-runtime and Singularity are installed but Docker isn't.
 
         json_dir = self._createTempDir()