diff --git a/.coveragerc b/.coveragerc index aae9c9a8..b7ddf072 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,4 +4,5 @@ omit = WDL/CLI.py WDL/runtime/backend/cli_subprocess.py WDL/runtime/backend/singularity.py + WDL/runtime/backend/podman.py relative_files = True diff --git a/WDL/runtime/backend/docker_swarm.py b/WDL/runtime/backend/docker_swarm.py index c3e97c97..0f44f23a 100644 --- a/WDL/runtime/backend/docker_swarm.py +++ b/WDL/runtime/backend/docker_swarm.py @@ -559,7 +559,7 @@ def chown(self, logger: logging.Logger, client: docker.DockerClient, success: bo ) script = f""" (find {paste} -type d -print0 && find {paste} -type f -print0) \ - | xargs -0 -P 10 chown -P {os.geteuid()}:{os.getegid()} + | xargs -0 -P 10 chown -Ph {os.geteuid()}:{os.getegid()} """.strip() volumes = {self.host_dir: {"bind": self.container_dir, "mode": "rw"}} logger.debug(_("post-task chown", script=script, volumes=volumes)) diff --git a/WDL/runtime/backend/podman.py b/WDL/runtime/backend/podman.py new file mode 100644 index 00000000..e3323887 --- /dev/null +++ b/WDL/runtime/backend/podman.py @@ -0,0 +1,193 @@ +import os +import time +import shlex +import logging +import threading +import subprocess +from typing import List, Callable, Optional +from ...Error import InputError, RuntimeError +from ..._util import StructuredLogMessage as _ +from .. import config +from ..error import DownloadFailed +from .cli_subprocess import SubprocessBase + + +class PodmanContainer(SubprocessBase): + """ + podman task runtime based on cli_subprocess.SubprocessBase + """ + + _tempdir: Optional[str] = None + _pull_lock: threading.Lock = threading.Lock() + _pulled_images = set() + + @classmethod + def global_init(cls, cfg: config.Loader, logger: logging.Logger) -> None: + podman_version_cmd = ["podman", "--version"] + if os.geteuid(): + podman_version_cmd = ["sudo", "-n"] + podman_version_cmd + + try: + podman_version = subprocess.run( + podman_version_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + except subprocess.CalledProcessError as cpe: + logger.error(_(" ".join(podman_version_cmd), stderr=cpe.stderr.strip().split("\n"))) + raise RuntimeError( + "Unable to check `sudo podman --version`; verify Podman installation" + " and no-password sudo (or run miniwdl as root)" + if os.geteuid() + else "Unable to check `podman --version`; verify Podman installation" + ) from None + + logger.notice( # pyre-ignore + _( + "Podman runtime initialized (BETA)", + podman_version=podman_version.stdout.strip(), + ) + ) + + @property + def cli_name(self) -> str: + return "podman" + + def _cli_invocation(self, logger: logging.Logger) -> List[str]: + """ + Formulate `podman run` command-line invocation + """ + image = self._podman_pull(logger) + + ans = ["podman"] + if os.geteuid(): + ans = ["sudo", "-n"] + ans + ans += [ + "run", + "--rm", + "--workdir", + os.path.join(self.container_dir, "work"), + ] + + cpu = self.runtime_values.get("cpu", 0) + if cpu > 0: + ans += ["--cpus", str(cpu)] + memory_limit = self.runtime_values.get("memory_limit", 0) + if memory_limit > 0: + ans += ["--memory", str(memory_limit)] + + mounts = self.prepare_mounts() + logger.info( + _( + "podman invocation", + args=" ".join(shlex.quote(s) for s in (ans + [image])), + binds=len(mounts), + ) + ) + for (container_path, host_path, writable) in mounts: + if ":" in (container_path + host_path): + raise InputError("Podman input filenames cannot contain ':'") + ans.append("-v") + bind_arg = f"{host_path}:{container_path}" + if not writable: + bind_arg += ":ro" + ans.append(bind_arg) + ans.append(image) + _sudo_canary() + return ans + + def _podman_pull(self, logger: logging.Logger) -> str: + """ + Ensure the needed docker image is cached by podman. Use a global lock so we'll only + download it once, even if used by many parallel tasks all starting at the same time. + """ + image = self.runtime_values.get( + "docker", self.cfg.get_dict("task_runtime", "defaults")["docker"] + ) + t0 = time.time() + with self._pull_lock: + t1 = time.time() + + if image in self._pulled_images: + logger.info(_("podman image already pulled", image=image)) + else: + _sudo_canary() + podman_pull_cmd = ["podman", "pull", image] + if os.geteuid(): + podman_pull_cmd = ["sudo", "-n"] + podman_pull_cmd + logger.info(_("begin podman pull", command=" ".join(podman_pull_cmd))) + try: + subprocess.run( + podman_pull_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + except subprocess.CalledProcessError as cpe: + logger.error( + _( + "podman pull failed", + stderr=cpe.stderr.strip().split("\n"), + stdout=cpe.stdout.strip().split("\n"), + ) + ) + raise DownloadFailed(image) from None + self._pulled_images.add(image) + + # TODO: log image ID? + logger.notice( # pyre-ignore + _( + "podman pull", + image=image, + seconds_waited=int(t1 - t0), + seconds_pulling=int(time.time() - t1), + ) + ) + return image + + def _run(self, logger: logging.Logger, terminating: Callable[[], bool], command: str) -> int: + """ + Override to chown working directory + """ + _sudo_canary() + try: + return super()._run(logger, terminating, command) + finally: + if os.geteuid(): + try: + subprocess.run( + [ + "sudo", + "-n", + "chown", + "-RPh", + f"{os.geteuid()}:{os.getegid()}", + self.host_work_dir(), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + except subprocess.CalledProcessError as cpe: + logger.error(_("post-task chown failed", error=cpe.stderr.strip().split("\n"))) + + +def _sudo_canary(): + if os.geteuid(): + try: + subprocess.run( + ["sudo", "-n", "id"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + check=True, + ) + except subprocess.SubprocessError: + raise RuntimeError( + "passwordless sudo expired (required for Podman)" + "; see miniwdl/podman documentation for workarounds" + ) diff --git a/WDL/runtime/backend/singularity.py b/WDL/runtime/backend/singularity.py index 961d8a61..e7d7f402 100644 --- a/WDL/runtime/backend/singularity.py +++ b/WDL/runtime/backend/singularity.py @@ -6,7 +6,7 @@ import threading import subprocess from typing import List, Callable, Optional -from ...Error import InputError +from ...Error import InputError, RuntimeError from ..._util import StructuredLogMessage as _ from ..._util import rmtree_atomic from .. import config @@ -34,14 +34,15 @@ def global_init(cls, cfg: config.Loader, logger: logging.Logger) -> None: universal_newlines=True, ) except: - assert False, "Unable to check `singularity --version`; verify Singularity installation" - logger.warning( + raise RuntimeError( + "Unable to check `singularity --version`; verify Singularity installation" + ) + logger.notice( # pyre-ignore _( - "Singularity runtime is experimental; use with caution", - version=singularity_version.stdout.strip(), + "Singularity runtime initialized (BETA)", + singularity_version=singularity_version.stdout.strip(), ) ) - pass @property def cli_name(self) -> str: diff --git a/WDL/runtime/config.py b/WDL/runtime/config.py index 7f1ac944..7035c952 100644 --- a/WDL/runtime/config.py +++ b/WDL/runtime/config.py @@ -363,6 +363,11 @@ def default_plugins() -> "Dict[str,List[importlib_metadata.EntryPoint]]": name="singularity", value="WDL.runtime.backend.singularity:SingularityContainer", ), + importlib_metadata.EntryPoint( + group="miniwdl.plugin.container_backend", + name="podman", + value="WDL.runtime.backend.podman:PodmanContainer", + ), ], "cache_backend": [ importlib_metadata.EntryPoint( diff --git a/tests/podman.t b/tests/podman.t new file mode 100644 index 00000000..7c480e3c --- /dev/null +++ b/tests/podman.t @@ -0,0 +1,38 @@ +#!/bin/bash +# bash-tap tests for miniwdl's Podman task runtime. Must run under sudo, with `podman` available. +set -o pipefail + +cd "$(dirname $0)/.." +SOURCE_DIR="$(pwd)" + +BASH_TAP_ROOT="tests/bash-tap" +source tests/bash-tap/bash-tap-bootstrap + +export PYTHONPATH="$SOURCE_DIR:$PYTHONPATH" +miniwdl="python3 -m WDL" + +if [[ -z $TMPDIR ]]; then + TMPDIR=/tmp +fi +DN=$(mktemp -d "${TMPDIR}/miniwdl_runner_tests_XXXXXX") +DN=$(realpath "$DN") +cd $DN +echo "$DN" + +plan tests 3 + +export MINIWDL__SCHEDULER__CONTAINER_BACKEND=podman + +$miniwdl run_self_test --dir "$DN" +is "$?" "0" "run_self_test" + +git clone --depth=1 https://github.com/broadinstitute/viral-pipelines.git +cd viral-pipelines + +$miniwdl run pipes/WDL/workflows/assemble_denovo.wdl \ + --path pipes/WDL/tasks --dir "$DN/assemble_denovo/." --verbose \ + -i test/input/WDL/test_inputs-assemble_denovo-local.json +is "$?" "0" "assemble_denovo success" + +is "$(find "$DN/assemble_denovo" | xargs -n 1 stat -c %u | sort | uniq)" "$(id -u)" \ + "assemble_denovo artifacts all owned by $(whoami)"