From 919a7dcac11cbbe632fd395ea475776a8ebfe2c4 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 16 Jun 2023 12:15:59 +0200 Subject: [PATCH] upgrade to cwltool 3.1.20230601100705 (#4500) Highlights: `--leave-tmpdir` includes the input staging directories by @mr-c in https://github.com/common-workflow-language/cwltool/pull/1674 https://github.com/common-workflow-language/cwltool/pull/1840 use importlib instead of the deprecated pkg_resources by @mr-c in https://github.com/common-workflow-language/cwltool/pull/1844 --- requirements-cwl.txt | 2 +- src/toil/cwl/__init__.py | 12 ++++++++---- src/toil/cwl/cwltoil.py | 25 +++++++++++++++---------- src/toil/test/cwl/cwlTest.py | 2 +- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/requirements-cwl.txt b/requirements-cwl.txt index 782afa8ab8..fdb54f2d6a 100644 --- a/requirements-cwl.txt +++ b/requirements-cwl.txt @@ -1,4 +1,4 @@ -cwltool==3.1.20230425144158 +cwltool==3.1.20230601100705 schema-salad>=8.4.20230128170514,<9 galaxy-tool-util ruamel.yaml>=0.15,<=0.17.21 diff --git a/src/toil/cwl/__init__.py b/src/toil/cwl/__init__.py index 3a8acb425b..bf43d3142d 100644 --- a/src/toil/cwl/__init__.py +++ b/src/toil/cwl/__init__.py @@ -15,15 +15,17 @@ from functools import lru_cache from pkg_resources import DistributionNotFound, get_distribution + try: # Setuptools 66+ will raise this if any package on the system has a version that isn't PEP440. # See https://github.com/pypa/setuptools/issues/3772 - from setuptools.extern.packaging.version import InvalidVersion # type: ignore + from setuptools.extern.packaging.version import InvalidVersion # type: ignore except ImportError: # It's not clear that this exception is really part fo the public API, so fake it. - class InvalidVersion(Exception): # type: ignore + class InvalidVersion(Exception): # type: ignore pass + from toil.version import cwltool_version logger = logging.getLogger(__name__) @@ -47,8 +49,10 @@ def check_cwltool_version() -> None: except DistributionNotFound: logger.debug("cwltool is not installed.") except InvalidVersion as e: - logger.warning(f"Could not determine the installed version of cwltool because a package " - f"with an unacceptable version is installed: {e}") + logger.warning( + f"Could not determine the installed version of cwltool because a package " + f"with an unacceptable version is installed: {e}" + ) check_cwltool_version() diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py index 31b3861641..3a77218ea7 100644 --- a/src/toil/cwl/cwltoil.py +++ b/src/toil/cwl/cwltoil.py @@ -60,10 +60,10 @@ import cwltool.builder import cwltool.command_line_tool import cwltool.context +import cwltool.cwlprov import cwltool.job import cwltool.load_tool import cwltool.main -import cwltool.provenance import cwltool.resolver import schema_salad.ref_resolver from cwltool.loghandler import _logger as cwllogger @@ -85,9 +85,9 @@ ) from cwltool.stdfsaccess import StdFsAccess, abspath from cwltool.utils import ( - DirectoryType, CWLObjectType, CWLOutputType, + DirectoryType, adjustDirObjs, aslist, downloadHttpFile, @@ -110,6 +110,7 @@ download_structure, visit_cwl_class_and_reduce, ) +from toil.exceptions import FailedJobsException from toil.fileStores import FileID from toil.fileStores.abstractFileStore import AbstractFileStore from toil.job import AcceleratorRequirement, Job, Promise, Promised, unwrap @@ -119,8 +120,6 @@ from toil.lib.threading import ExceptionalThread from toil.statsAndLogging import DEFAULT_LOGLEVEL from toil.version import baseVersion -from toil.exceptions import FailedJobsException - logger = logging.getLogger(__name__) @@ -2015,7 +2014,8 @@ def _realpath( f.close() # Import it and pack up the file ID so we can turn around and export it. file_id_or_contents = ( - "toilfile:" + toil.import_file(f.name, symlink=False).pack() + "toilfile:" + + toil.import_file(f.name, symlink=False).pack() ) if file_id_or_contents.startswith("toildir:"): @@ -3625,7 +3625,7 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int: loading_context = cwltool.main.setup_loadingContext(None, runtime_context, options) if options.provenance: - research_obj = cwltool.provenance.ResearchObject( + research_obj = cwltool.cwlprov.ro.ResearchObject( temp_prefix_ro=options.tmp_outdir_prefix, orcid=options.orcid, full_name=options.cwl_full_name, @@ -3708,8 +3708,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int: document_loader = loading_context.loader if options.provenance and runtime_context.research_obj: - runtime_context.research_obj.packed_workflow( - cwltool.main.print_pack(loading_context, uri) + cwltool.cwlprov.writablebagfile.packed_workflow( + runtime_context.research_obj, + cwltool.main.print_pack(loading_context, uri), ) try: @@ -3879,7 +3880,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int: toilStageFiles(toil, outobj, outdir, destBucket=options.destBucket) if runtime_context.research_obj is not None: - runtime_context.research_obj.create_job(outobj, True) + cwltool.cwlprov.writablebagfile.create_job( + runtime_context.research_obj, outobj, True + ) def remove_at_id(doc: Any) -> None: if isinstance(doc, MutableMapping): @@ -3906,7 +3909,9 @@ def remove_at_id(doc: Any) -> None: workflowobj, document_loader, uri ) runtime_context.research_obj.generate_snapshot(prov_dependencies) - runtime_context.research_obj.close(options.provenance) + cwltool.cwlprov.writablebagfile.close_ro( + runtime_context.research_obj, options.provenance + ) if not options.destBucket and options.compute_checksum: visit_class( diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py index c4e0498f49..679b8065af 100644 --- a/src/toil/test/cwl/cwlTest.py +++ b/src/toil/test/cwl/cwlTest.py @@ -37,6 +37,7 @@ from toil.cwl.utils import (download_structure, visit_cwl_class_and_reduce, visit_top_cwl_class) +from toil.exceptions import FailedJobsException from toil.fileStores import FileID from toil.fileStores.abstractFileStore import AbstractFileStore from toil.lib.aws import zone_to_region @@ -61,7 +62,6 @@ needs_torque, needs_wes_server, slow) -from toil.exceptions import FailedJobsException from toil.test.provisioners.aws.awsProvisionerTest import \ AbstractAWSAutoscaleTest from toil.test.provisioners.clusterTest import AbstractClusterTest