upgrade to cwltool 3.1.20230601100705 (#4500)

Highlights:

`--leave-tmpdir` includes the input staging directories by @mr-c in common-workflow-language/cwltool#1674 common-workflow-language/cwltool#1840
use importlib instead of the deprecated pkg_resources by @mr-c in common-workflow-language/cwltool#1844

requirements-cwl.txt

@@ -1,4 +1,4 @@
-cwltool==3.1.20230425144158
+cwltool==3.1.20230601100705
 schema-salad>=8.4.20230128170514,<9
 galaxy-tool-util
 ruamel.yaml>=0.15,<=0.17.21

src/toil/cwl/__init__.py

@@ -15,15 +15,17 @@
 from functools import lru_cache
 
 from pkg_resources import DistributionNotFound, get_distribution
+
 try:
     # Setuptools 66+ will raise this if any package on the system has a version that isn't PEP440.
     # See https://github.com/pypa/setuptools/issues/3772
-    from setuptools.extern.packaging.version import InvalidVersion # type: ignore
+    from setuptools.extern.packaging.version import InvalidVersion  # type: ignore
 except ImportError:
     # It's not clear that this exception is really part fo the public API, so fake it.
-    class InvalidVersion(Exception): # type: ignore
+    class InvalidVersion(Exception):  # type: ignore
         pass
 
+
 from toil.version import cwltool_version
 
 logger = logging.getLogger(__name__)
@@ -47,8 +49,10 @@ def check_cwltool_version() -> None:
     except DistributionNotFound:
         logger.debug("cwltool is not installed.")
     except InvalidVersion as e:
-        logger.warning(f"Could not determine the installed version of cwltool because a package "
-                       f"with an unacceptable version is installed: {e}")
+        logger.warning(
+            f"Could not determine the installed version of cwltool because a package "
+            f"with an unacceptable version is installed: {e}"
+        )
 
 
 check_cwltool_version()

src/toil/cwl/cwltoil.py

@@ -60,10 +60,10 @@
 import cwltool.builder
 import cwltool.command_line_tool
 import cwltool.context
+import cwltool.cwlprov
 import cwltool.job
 import cwltool.load_tool
 import cwltool.main
-import cwltool.provenance
 import cwltool.resolver
 import schema_salad.ref_resolver
 from cwltool.loghandler import _logger as cwllogger
@@ -85,9 +85,9 @@
 )
 from cwltool.stdfsaccess import StdFsAccess, abspath
 from cwltool.utils import (
-    DirectoryType,
     CWLObjectType,
     CWLOutputType,
+    DirectoryType,
     adjustDirObjs,
     aslist,
     downloadHttpFile,
@@ -110,6 +110,7 @@
     download_structure,
     visit_cwl_class_and_reduce,
 )
+from toil.exceptions import FailedJobsException
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import AcceleratorRequirement, Job, Promise, Promised, unwrap
@@ -119,8 +120,6 @@
 from toil.lib.threading import ExceptionalThread
 from toil.statsAndLogging import DEFAULT_LOGLEVEL
 from toil.version import baseVersion
-from toil.exceptions import FailedJobsException
-
 
 logger = logging.getLogger(__name__)
 
@@ -2015,7 +2014,8 @@ def _realpath(
                             f.close()
                             # Import it and pack up the file ID so we can turn around and export it.
                             file_id_or_contents = (
-                                "toilfile:" + toil.import_file(f.name, symlink=False).pack()
+                                "toilfile:"
+                                + toil.import_file(f.name, symlink=False).pack()
                             )
 
                     if file_id_or_contents.startswith("toildir:"):
@@ -3625,7 +3625,7 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
     loading_context = cwltool.main.setup_loadingContext(None, runtime_context, options)
 
     if options.provenance:
-        research_obj = cwltool.provenance.ResearchObject(
+        research_obj = cwltool.cwlprov.ro.ResearchObject(
             temp_prefix_ro=options.tmp_outdir_prefix,
             orcid=options.orcid,
             full_name=options.cwl_full_name,
@@ -3708,8 +3708,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
             document_loader = loading_context.loader
 
             if options.provenance and runtime_context.research_obj:
-                runtime_context.research_obj.packed_workflow(
-                    cwltool.main.print_pack(loading_context, uri)
+                cwltool.cwlprov.writablebagfile.packed_workflow(
+                    runtime_context.research_obj,
+                    cwltool.main.print_pack(loading_context, uri),
                 )
 
             try:
@@ -3879,7 +3880,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
         toilStageFiles(toil, outobj, outdir, destBucket=options.destBucket)
 
         if runtime_context.research_obj is not None:
-            runtime_context.research_obj.create_job(outobj, True)
+            cwltool.cwlprov.writablebagfile.create_job(
+                runtime_context.research_obj, outobj, True
+            )
 
             def remove_at_id(doc: Any) -> None:
                 if isinstance(doc, MutableMapping):
@@ -3906,7 +3909,9 @@ def remove_at_id(doc: Any) -> None:
                 workflowobj, document_loader, uri
             )
             runtime_context.research_obj.generate_snapshot(prov_dependencies)
-            runtime_context.research_obj.close(options.provenance)
+            cwltool.cwlprov.writablebagfile.close_ro(
+                runtime_context.research_obj, options.provenance
+            )
 
         if not options.destBucket and options.compute_checksum:
             visit_class(

src/toil/test/cwl/cwlTest.py

@@ -37,6 +37,7 @@
 from toil.cwl.utils import (download_structure,
                             visit_cwl_class_and_reduce,
                             visit_top_cwl_class)
+from toil.exceptions import FailedJobsException
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.lib.aws import zone_to_region
@@ -61,7 +62,6 @@
                        needs_torque,
                        needs_wes_server,
                        slow)
-from toil.exceptions import FailedJobsException
 from toil.test.provisioners.aws.awsProvisionerTest import \
     AbstractAWSAutoscaleTest
 from toil.test.provisioners.clusterTest import AbstractClusterTest