Merge branch 'main' into dependabot/pip/mypy-1.4.0

.github/workflows/ci-tests.yml

@@ -28,19 +28,12 @@ jobs:
 
   tox:
     name: Tox
-    runs-on: ubuntu-20.04  # 22.04 doesn't support Python 3.6
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9, 10, 11]
+        py-ver-minor: [8, 9, 10, 11, 12]
         step: [lint, unit, bandit, mypy]
-        exclude:
-          - py-ver-major: 3
-            py-ver-minor: 6
-            step: mypy
-          - py-ver-major: 3
-            py-ver-minor: 6
-            step: lint
 
     env:
       py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
@@ -65,6 +58,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.py-semver }}
+          allow-prereleases: true
           cache: pip
           cache-dependency-path: |
             requirements.txt

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 Style guide:
 - PEP-8 (as implemented by the `black` code formatting tool)
-- Python 3.6+ compatible code
+- Python 3.8+ compatible code
 - PEP-484 type hints
 
 The development is done using `git`, we encourage you to get familiar with it.

Makefile

@@ -25,7 +25,7 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py
-DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv \
+DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \
 	isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\
 	-rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
@@ -191,16 +191,6 @@ mypy: $(PYSOURCES)
 	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
 	MYPYPATH=$$MYPYPATH:mypy-stubs mypy $^
 
-mypy_3.6: $(filter-out setup.py gittagger.py,$(PYSOURCES))
-	if ! test -f $(shell python -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
-	then \
-		rm -Rf mypy-stubs/ruamel/yaml ; \
-		ln -s $(shell python -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-			mypy-stubs/ruamel/ ; \
-	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
-	MYPYPATH=$$MYPYPATH:mypy-stubs mypy --python-version 3.6 $^
-
-
 mypyc: $(PYSOURCES)
 	MYPYPATH=mypy-stubs CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
 		 && pytest -rs -vv ${PYTEST_EXTRA}
@@ -210,7 +200,8 @@ shellcheck: FORCE
 		cwltool-in-docker.sh
 
 pyupgrade: $(PYSOURCES)
-	pyupgrade --exit-zero-even-if-changed --py36-plus $^
+	pyupgrade --exit-zero-even-if-changed --py38-plus $^
+	auto-walrus $^
 
 release-test: check-python3 FORCE
 	git diff-index --quiet HEAD -- || ( echo You have uncommitted changes, please commit them and try again; false )

README.rst

@@ -52,7 +52,7 @@ and provide comprehensive validation of CWL
 files as well as provide other tools related to working with CWL.
 
 ``cwltool`` is written and tested for
-`Python <https://www.python.org/>`_ ``3.x {x = 6, 7, 8, 9, 10, 11}``
+`Python <https://www.python.org/>`_ ``3.x {x = 6, 8, 9, 10, 11}``
 
 The reference implementation consists of two packages.  The ``cwltool`` package
 is the primary Python module containing the reference implementation in the

cwltool/builder.py

@@ -532,7 +532,7 @@ def addsf(
                                     "Reference, a single format string, or a list of "
                                     "format strings. But the list cannot contain CWL "
                                     "Expressions or CWL Parameter References. List "
-                                    f"entry number {index+1} contains the following "
+                                    f"entry number {index + 1} contains the following "
                                     "unallowed CWL Parameter Reference or Expression: "
                                     f"{entry!r}."
                                 )

cwltool/checker.py

@@ -5,6 +5,7 @@
     Dict,
     Iterator,
     List,
+    Literal,
     MutableMapping,
     MutableSequence,
     Optional,
@@ -16,7 +17,6 @@
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno
 from schema_salad.utils import json_dumps
-from typing_extensions import Literal
 
 from .errors import WorkflowException
 from .loghandler import _logger
@@ -288,7 +288,9 @@ def static_checker(
             )
             + "\n"
             + SourceLine(sink, "type").makeError(
-                "  with sink '%s' of type %s" % (shortname(sink["id"]), json_dumps(sink["type"]))
+                "  with sink '{}' of type {}".format(
+                    shortname(sink["id"]), json_dumps(sink["type"])
+                )
             )
         )
         if extra_message is not None:
@@ -507,8 +509,7 @@ def get_step_id(field_id: str) -> str:
 
 
 def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool:
-    source_step = param_to_step.get(parm_id)
-    if source_step is not None:
+    if (source_step := param_to_step.get(parm_id)) is not None:
         if source_step.get("when") is not None:
             return True
     return False

cwltool/command_line_tool.py

@@ -693,7 +693,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None:
         for i, t2 in enumerate(ls):
             if not isinstance(t2, Mapping):
                 raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError(
-                    "Entry at index %s of listing is not a record, was %s" % (i, type(t2))
+                    f"Entry at index {i} of listing is not a record, was {type(t2)}"
                 )
 
             if "entry" not in t2:
@@ -715,7 +715,9 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None:
 
             if not isinstance(t2["entry"], Mapping):
                 raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError(
-                    "Entry at index %s of listing is not a record, was %s" % (i, type(t2["entry"]))
+                    "Entry at index {} of listing is not a record, was {}".format(
+                        i, type(t2["entry"])
+                    )
                 )
 
             if t2["entry"].get("class") not in ("File", "Directory"):
@@ -1454,10 +1456,18 @@ def collect_output(
                                         continue
                                     if isinstance(sfitem, str):
                                         sfitem = {"path": pathprefix + sfitem}
-                                    if not fs_access.exists(sfitem["path"]) and sf_required:
+                                    original_sfitem = copy.deepcopy(sfitem)
+                                    if (
+                                        not fs_access.exists(
+                                            cast(
+                                                str, cast(CWLObjectType, revmap(sfitem))["location"]
+                                            )
+                                        )
+                                        and sf_required
+                                    ):
                                         raise WorkflowException(
                                             "Missing required secondary file '%s'"
-                                            % (sfitem["path"])
+                                            % (original_sfitem["path"])
                                         )
                                     if "path" in sfitem and "location" not in sfitem:
                                         revmap(sfitem)

cwltool/context.py

@@ -12,6 +12,7 @@
     Dict,
     Iterable,
     List,
+    Literal,
     Optional,
     TextIO,
     Tuple,
@@ -22,7 +23,6 @@
 from schema_salad.avro.schema import Names
 from schema_salad.ref_resolver import Loader
 from schema_salad.utils import FetcherCallableType
-from typing_extensions import Literal
 
 from .mpi import MpiConfig
 from .pathmapper import PathMapper

cwltool/cwlprov/__init__.py

@@ -6,9 +6,7 @@
 import re
 import uuid
 from getpass import getuser
-from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
-
-from typing_extensions import TypedDict
+from typing import IO, Any, Callable, Dict, List, Optional, Tuple, TypedDict, Union
 
 
 def _whoami() -> Tuple[str, str]:
@@ -112,24 +110,26 @@ def _valid_orcid(orcid: Optional[str]) -> str:
         "oa:motivatedBy": Dict[str, str],
     },
 )
-Aggregate = TypedDict(
-    "Aggregate",
-    {
-        "uri": Optional[str],
-        "bundledAs": Optional[Dict[str, Any]],
-        "mediatype": Optional[str],
-        "conformsTo": Optional[Union[str, List[str]]],
-        "createdOn": Optional[str],
-        "createdBy": Optional[Dict[str, str]],
-    },
-    total=False,
-)
+
+
+class Aggregate(TypedDict, total=False):
+    """RO Aggregate class."""
+
+    uri: Optional[str]
+    bundledAs: Optional[Dict[str, Any]]
+    mediatype: Optional[str]
+    conformsTo: Optional[Union[str, List[str]]]
+    createdOn: Optional[str]
+    createdBy: Optional[Dict[str, str]]
+
+
 # Aggregate.bundledAs is actually type Aggregate, but cyclic definitions are not supported
-AuthoredBy = TypedDict(
-    "AuthoredBy",
-    {"orcid": Optional[str], "name": Optional[str], "uri": Optional[str]},
-    total=False,
-)
+class AuthoredBy(TypedDict, total=False):
+    """RO AuthoredBy class."""
+
+    orcid: Optional[str]
+    name: Optional[str]
+    uri: Optional[str]
 
 
 def checksum_copy(

cwltool/cwlrdf.py

@@ -51,7 +51,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
 
     for step, run, _ in qres:
         stdout.write(
-            '"%s" [label="%s"]\n' % (lastpart(step), f"{lastpart(step)} ({lastpart(run)})")
+            '"{}" [label="{}"]\n'.format(lastpart(step), f"{lastpart(step)} ({lastpart(run)})")
         )
 
     qres = cast(
@@ -170,7 +170,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
 
         if str(runtype) != "https://w3id.org/cwl/cwl#Workflow":
             stdout.write(
-                '"%s" [label="%s"]\n' % (dotname[step], urllib.parse.urldefrag(str(step))[1])
+                f'"{dotname[step]}" [label="{urllib.parse.urldefrag(str(step))[1]}"]\n'  # noqa: B907
             )
 
     if currentwf is not None:

cwltool/docker.py

@@ -47,7 +47,7 @@ def _get_docker_machine_mounts() -> List[str]:
                             "-t",
                             "vboxsf",
                         ],
-                        universal_newlines=True,
+                        text=True,
                     ).splitlines()
                 ]
     return __docker_machine_mounts

cwltool/docker_id.py

@@ -30,9 +30,7 @@ def check_output_and_strip(cmd: List[str]) -> Optional[str]:
     :return: Stripped string output of the command, or ``None`` if error
     """
     try:
-        result = subprocess.check_output(  # nosec
-            cmd, stderr=subprocess.STDOUT, universal_newlines=True
-        )
+        result = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)  # nosec
         return result.strip()
     except (OSError, subprocess.CalledProcessError, TypeError, AttributeError):
         # OSError is raised if command doesn't exist

cwltool/job.py

@@ -810,7 +810,7 @@ def run(
                 _logger.debug("%s error", container, exc_info=True)
                 if docker_is_req:
                     raise UnsupportedRequirement(
-                        "{} is required to run this tool: {}".format(container, str(err))
+                        f"{container} is required to run this tool: {str(err)}"
                     ) from err
                 else:
                     raise WorkflowException(

cwltool/load_tool.py

@@ -379,9 +379,7 @@ def resolve_and_validate_document(
     loadingContext = loadingContext.copy()
 
     if not isinstance(workflowobj, MutableMapping):
-        raise ValueError(
-            "workflowjobj must be a dict, got '{}': {}".format(type(workflowobj), workflowobj)
-        )
+        raise ValueError(f"workflowjobj must be a dict, got {type(workflowobj)!r}: {workflowobj}")
 
     jobobj = None
     if "cwl:tool" in workflowobj:

cwltool/singularity.py

@@ -43,9 +43,7 @@ def get_version() -> Tuple[List[int], str]:
     global _SINGULARITY_VERSION  # pylint: disable=global-statement
     global _SINGULARITY_FLAVOR  # pylint: disable=global-statement
     if _SINGULARITY_VERSION is None:
-        version_output = check_output(  # nosec
-            ["singularity", "--version"], universal_newlines=True
-        ).strip()
+        version_output = check_output(["singularity", "--version"], text=True).strip()  # nosec
 
         version_match = re.match(r"(.+) version ([0-9\.]+)", version_output)
         if version_match is None:

cwltool/software_requirements.py

@@ -91,9 +91,8 @@ def build_job_script(self, builder: "Builder", command: List[str]) -> str:
             resolution_config_dict=resolution_config_dict,
             conf_file=self.dependency_resolvers_config_file,
         )
-        dependencies = get_dependencies(builder)
-        handle_dependencies = ""  # str
-        if dependencies:
+        handle_dependencies: str = ""
+        if dependencies := get_dependencies(builder):
             handle_dependencies = "\n".join(
                 tool_dependency_manager.dependency_shell_commands(
                     dependencies, job_directory=builder.tmpdir