Enable FUSE for privileged Toil clusters (#4824)

* Add option for privileged clusters and enable privileges for toil-managed clusters * Fix syntax error and add back namespace rules * packages might be broken * Dependencies * Move apt clean * Create test image * Create test image 2 * Try just creating the base docker image * test image creation, typo * Try focal debian package * Try the last docker build command * remove nontoil makefile dependencies to test * Successfully build docker images at least for amd64 * Remove unprivileged fuse mount code * Bring back rest of docker builds * Remove unnecessary env var in dockerfile * Fix setuptools and virtualenv to some version and revert whitespace * Apply suggestions from code review Co-authored-by: Adam Novak <[email protected]> * Move SINGULARITY_CACHEDIR comment * Formatting and move strtobool * Reflect moved functions for imports * Remove debug_mute flag and print debugging statement outside instead --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Adam Novak <[email protected]>
DataBiosphere · Apr 26, 2024 · c566f31 · c566f31
1 parent 8438f73
commit c566f31
Show file tree

Hide file tree

Showing 19 changed files with 140 additions and 67 deletions.
diff --git a/Makefile b/Makefile
@@ -239,7 +239,7 @@ push_docker: docker
 
 load_docker: docker
 	cd docker ; docker buildx build --platform $(arch) --load --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../.docker-cache/toil -f Dockerfile .
-	cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile . 
+	cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile .
 	cd dashboard/grafana ; docker buildx build --platform $(arch) --load --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/grafana -f Dockerfile .
 	cd dashboard/mtail ; docker buildx build --platform $(arch) --load --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/mtail -f Dockerfile .
 

diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
@@ -24,15 +24,17 @@
 python = f'python{sys.version_info[0]}.{sys.version_info[1]}'
 pip = f'{python} -m pip'
 
-
-dependencies = ' '.join(['libffi-dev',  # For client side encryption for extras with PyNACL
+# Debian and Ubuntu don't package ensurepip by default so the python-venv package must be installed
+python_packages = {'python3.8': ['python3.8-distutils', 'python3.8-venv'],
+                   'python3.9': ['python3.9-distutils', 'python3.9-venv'],
+                   'python3.10': ['python3.10-distutils', 'python3.10-venv'],
+                   'python3.11': ['python3.11-distutils', 'python3.11-venv'],
+                   'python3.12': ['python3.12-distutils', 'python3.12-venv']}
+
+dependencies = ' '.join(python_packages[python] +
+                        ['libffi-dev',  # For client side encryption for extras with PyNACL
                          python,
                          f'{python}-dev',
-                         'python3.8-distutils' if python == 'python3.8' else '',
-                         'python3.9-distutils' if python == 'python3.9' else '',
-                         'python3.10-distutils' if python == 'python3.10' else '',
-                         'python3.11-distutils' if python == 'python3.11' else '',
-                         'python3.12-distutils' if python== 'python3.12' else '',
                          'python3-pip',
                          'libssl-dev',
                          'wget',
@@ -62,6 +64,11 @@
                          'libapr1',
                          # Dependencies for singularity
                          'containernetworking-plugins',
+                         'libfuse2',
+                         'fuse',
+                         'fuse2fs',
+                         'uidmap',
+                         'squashfs-tools-ng',
                          # Dependencies for singularity on kubernetes
                          'tzdata'])
 
@@ -93,9 +100,6 @@ def heredoc(s):
 
     RUN if [ -z "$TARGETARCH" ] ; then echo "Specify a TARGETARCH argument to build this container"; exit 1; fi
 
-    # make sure we don't use too new a version of setuptools (which can get out of sync with poetry and break things)
-    ENV SETUPTOOLS_USE_DISTUTILS=stdlib
-
     # Try to avoid "Failed to fetch ...  Undetermined Error" from apt
     # See <https://stackoverflow.com/a/66523384>
     RUN printf 'Acquire::http::Pipeline-Depth "0";\\nAcquire::http::No-Cache=True;\\nAcquire::BrokenProxy=true;\\n' >/etc/apt/apt.conf.d/99fixbadproxy
@@ -119,7 +123,7 @@ def heredoc(s):
         if [ $TARGETARCH = amd64 ] ; then DEBIAN_FRONTEND=noninteractive apt-get -y install mesos ; mesos-agent --help >/dev/null ; fi && \
         apt-get clean && \
         rm -rf /var/lib/apt/lists/*
-    
+
     # Install a particular old Debian Sid Singularity from somewhere.
     # It's 3.10, which is new enough to use cgroups2, but it needs a newer libc
     # than Ubuntu 20.04 ships. So we need a 22.04+ base.
@@ -141,6 +145,15 @@ def heredoc(s):
     # <https://github.com/apptainer/singularity/issues/6113#issuecomment-901897566>).
     #
     # So we need to make sure to install a downgraded squashfs first.
+    #
+    # TODO: Singularity has since resolved this on their end
+    # https://github.com/sylabs/singularity/pull/267
+    # This works by checking that the UID of the caller is not root
+    # In a Kubernetes pod, the default setup will have UID 0 even if the pod is unprivileged
+    # https://github.com/sylabs/singularity/issues/2727
+    # It is possible to avoid this by changing the user of the pod (such as runAsUser: 1000)
+    # but this may cause permission issues, ex: changed read/write permissions
+    # so the downgraded squashfs stays, but options for updated squashfs are possible
     ADD extra-debs.tsv /etc/singularity/extra-debs.tsv
     RUN wget -q "$(cat /etc/singularity/extra-debs.tsv | grep "^squashfs-tools.$TARGETARCH" | cut -f4)" && \
         dpkg -i squashfs-tools_*.deb && \
@@ -166,18 +179,20 @@ def heredoc(s):
     RUN chmod 777 /usr/bin/waitForKey.sh && chmod 777 /usr/bin/customDockerInit.sh && chmod 777 /usr/bin/singularity
 
     # The stock pip is too old and can't install from sdist with extras
-    RUN curl -sS https://bootstrap.pypa.io/get-pip.py | {python}
-
-    # Default setuptools is too old
-    RUN {pip} install --upgrade setuptools==59.7.0
+    RUN {python} -m ensurepip --upgrade
 
     # Include virtualenv, as it is still the recommended way to deploy pipelines
-    RUN {pip} install --upgrade virtualenv==20.0.17
+    RUN {pip} install --upgrade virtualenv==20.25.1
 
     # Install s3am (--never-download prevents silent upgrades to pip, wheel and setuptools)
+    # Install setuptools within the virtual environment to properly access distutils due to PEP 632 and gh-95299 in Python 3.12 release notes
+    # https://docs.python.org/3/whatsnew/3.12.html#summary-release-highlights
     RUN virtualenv --python {python} --never-download /home/s3am \
+        && /home/s3am/bin/pip install setuptools \
         && /home/s3am/bin/pip install s3am==2.0 \
         && ln -s /home/s3am/bin/s3am /usr/local/bin/
+    
+    RUN {pip} install --upgrade setuptools==69.2.0
 
     # Fix for https://issues.apache.org/jira/browse/MESOS-3793
     ENV MESOS_LAUNCHER=posix

diff --git a/docker/extra-debs.tsv b/docker/extra-debs.tsv
@@ -1,5 +1,5 @@
 #Package	Architecture	SHA256	URL	URL	...
-squashfs-tools	amd64	883f4f4de16bd6a1551a7ef43cdb58f9f103fdc7f68b480bb24c60d569a463ae	https://snapshot.debian.org/archive/debian/20150805T214000Z/pool/main/s/squashfs-tools/squashfs-tools_4.2%2B20130409-2.1_amd64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_amd64.deb
-squashfs-tools	arm64	1d2e9e9dc8428c13e7af967f7f0984c35fd136b33e81a9216e80fdb6a8f562f1	https://snapshot.debian.org/archive/debian/20150805T214000Z/pool/main/s/squashfs-tools/squashfs-tools_4.2%2B20130409-2.1_arm64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_arm64.deb
-singularity-container	amd64	1e8b5aed949cba5d9ace3f242323a4b25f224027026d6a5f598f18ccf5dfc79c	https://snapshot.debian.org/archive/debian/20221013T214821Z/pool/main/s/singularity-container/singularity-container_3.10.3%2Bds1-1_amd64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_amd64.deb
-singularity-container	arm64	b439a7c4f6d8db7d4dcd2f0169e5894a659b842c26fccad8f411b2b748678353	https://snapshot.debian.org/archive/debian/20221013T214821Z/pool/main/s/singularity-container/singularity-container_3.10.3%2Bds1-1_arm64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_arm64.deb
+squashfs-tools	amd64	883f4f4de16bd6a1551a7ef43cdb58f9f103fdc7f68b480bb24c60d569a463ae	https://snapshot.debian.org/archive/debian/20150805T214000Z/pool/main/s/squashfs-tools/squashfs-tools_4.2%2B20130409-2.1_amd64.deb
+squashfs-tools	arm64	1d2e9e9dc8428c13e7af967f7f0984c35fd136b33e81a9216e80fdb6a8f562f1	https://snapshot.debian.org/archive/debian/20150805T214000Z/pool/main/s/squashfs-tools/squashfs-tools_4.2%2B20130409-2.1_arm64.deb
+singularity-container	amd64	13e4526bd6673283cec49bad245f203e9ab7d1a357930408c1ce88a0af5445d6	https://snapshot.debian.org/archive/debian/20240407T144320Z/pool/main/s/singularity-container/singularity-container_4.1.2%2Bds1-1_amd64.deb
+singularity-container	arm64	7e1aef5a965f7531f6234c760854b285e2f864f51163f1cd0d11b26854ef0dca	https://snapshot.debian.org/archive/debian/20240407T144320Z/pool/main/s/singularity-container/singularity-container_4.1.2%2Bds1-1_arm64.deb
diff --git a/docs/contributing/contributing.rst b/docs/contributing/contributing.rst
@@ -204,7 +204,8 @@ Here is a general workflow (similar instructions apply when using Docker Hub):
         $ make docker
 
    to automatically build a docker image that can now be uploaded to
-   your personal `Quay`_ account. If you have not installed Toil source
+   your personal `Quay`_ account. On Docker Desktop, containerd `may have to be enabled <https://docs.docker.com/desktop/containerd/#enable-the-containerd-image-store>`_.
+   If you have not installed Toil source
    code yet see :ref:`buildFromSource`.
 
 #. If it's not already you will need Docker installed and need
@@ -310,6 +311,31 @@ with ``worker`` to get shell access in your worker.
     if Docker can't find the file/directory on the host it will silently fail and mount
     in an empty directory.
 
+
+.. admonition:: Enabling FUSE
+
+    When running toil-wdl-runner with Singularity, Singularity will decompress images to sandbox directories
+    by default. This can take time if a workflow has lots of images. To avoid this, access to FUSE can be given
+    to the Docker container at startup. There are 2 main ways to do this. Either run all the Docker
+    containers in privileged mode::
+
+        docker run \
+            -d \
+            --name=toil_leader \
+            --privileged \
+            quay.io/ucsc_cgl/toil:6.2.0
+
+    Or pass through the ``/dev/fuse`` device node into the container::
+
+        docker run \
+            -d \
+            --name=toil_leader \
+            --device=/dev/fuse \
+            quay.io/ucsc_cgl/toil:6.2.0
+
+    toil-wdl-runner will handle the logic from there.
+
+
 .. _Quay: https://quay.io/
 .. _Docker Hub: https://hub.docker.com/
 

diff --git a/docs/running/cliOptions.rst b/docs/running/cliOptions.rst
@@ -239,6 +239,11 @@ levels in toil are based on priority from the logging module:
   --kubernetesPodTimeout KUBERNETES_POD_TIMEOUT
                         Seconds to wait for a scheduled Kubernetes pod to
                         start running. (default: 120s)
+  --kubernetesPrivileged BOOL
+                        Whether to allow Kubernetes pods to run as privileged. This can be
+                        used to enable FUSE mounts for faster runtimes with Singularity.
+                        When launching Toil-managed clusters, this will be set to true by --allowFuse.
+                        (default: False)
   --awsBatchRegion AWS_BATCH_REGION
                         The AWS region containing the AWS Batch queue to submit
                         to.

diff --git a/docs/running/cloud/clusterUtils.rst b/docs/running/cloud/clusterUtils.rst
@@ -125,6 +125,11 @@ exist yet, Toil will create it for you.
                         to override the default value from ``--nodeStorage`` for the
                         specified nodeType(s). This is useful for heterogeneous jobs
                         where some tasks require much more disk than others.
+  --allowFuse BOOL
+                        Whether to allow FUSE mounts for faster runtimes with Singularity.
+                        Note: This will result in the Toil container running as privileged.
+                        For Kubernetes, pods will be asked to run as privileged. If this is not
+                        allowed, Singularity containers will use sandbox directories instead.
 
 **Logging Options**
 

diff --git a/docs/running/cloud/kubernetes.rst b/docs/running/cloud/kubernetes.rst
@@ -188,6 +188,8 @@ To configure your workflow to run on Kubernetes, you will have to configure seve
 
 #. ``TOIL_KUBERNETES_OWNER`` **should** be set to the username of the user running the Toil workflow. The jobs that Toil creates will include this username, so they can be more easily recognized, and cleaned up by the user if anything happens to the Toil leader. In this example we are using ``demo-user``.
 
+#. ``TOIL_KUBERNETES_PRIVILEGED`` **can** be set to True or False. When true, this allows pods to run as privileged, enabling FUSE mounts for Singularity for faster runtimes. If this is not set to true, Singularity will extract images to sandbox directories. This is unset/False by default except in Toil-managed clusters.
+
 Note that Docker containers cannot be run inside of unprivileged Kubernetes pods (which are themselves containers). The Docker daemon does not (yet) support this. Other tools, such as Singularity in its user-namespace mode, are able to run containers from within containers. If using Singularity to run containerized tools, and you want downloaded container images to persist between Toil jobs, some setup may be required:
 
 **On non-Toil managed clusters:**

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -27,3 +27,4 @@ lxml
 docutils>=0.16,<0.22
 pyupgrade
 pytest-xdist
+build
diff --git a/src/toil/batchSystems/kubernetes.py b/src/toil/batchSystems/kubernetes.py
@@ -47,6 +47,8 @@
                     cast,
                     overload)
 
+from toil.lib.conversions import opt_strtobool
+
 if sys.version_info < (3, 10):
     from typing_extensions import ParamSpec
 else:
@@ -83,7 +85,7 @@
                                V1SecretVolumeSource,
                                V1Toleration,
                                V1Volume,
-                               V1VolumeMount)
+                               V1VolumeMount, V1SecurityContext)
 from kubernetes.client.api_client import ApiClient
 from kubernetes.client.exceptions import ApiException
 from kubernetes.config.config_exception import ConfigException
@@ -878,14 +880,20 @@ def mount_host_path(volume_name: str, host_path: str, mount_path: str, create: b
 
         # Make a container definition
         container = V1Container(command=command_list,
-                                                  image=self.docker_image,
-                                                  name="runner-container",
-                                                  resources=resources,
-                                                  volume_mounts=mounts)
+                                image=self.docker_image,
+                                name="runner-container",
+                                resources=resources,
+                                volume_mounts=mounts)
+
+        # In case security context rules are not allowed to be set, we only apply
+        # a security context at all if we need to turn on privileged mode.
+        if self.config.kubernetes_privileged:
+            container.security_context = V1SecurityContext(privileged=self.config.kubernetes_privileged)
+
         # Wrap the container in a spec
         pod_spec = V1PodSpec(containers=[container],
-                                               volumes=volumes,
-                                               restart_policy="Never")
+                             volumes=volumes,
+                             restart_policy="Never")
         # Tell the spec where to land
         placement.apply(pod_spec)
 
@@ -1880,6 +1888,10 @@ def add_options(cls, parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
         parser.add_argument("--kubernetesPodTimeout", dest="kubernetes_pod_timeout", default=120, env_var="TOIL_KUBERNETES_POD_TIMEOUT", type=float,
                             help="Seconds to wait for a scheduled Kubernetes pod to start running.  "
                                  "(default: %(default)s)")
+        parser.add_argument("--kubernetesPrivileged", dest="kubernetes_privileged", default=False, env_var="TOIL_KUBERNETES_PRIVILEGED", type=opt_strtobool,
+                            help="Whether to ask worker pods to run in privileged mode. This should be used to access "
+                                 "privileged operations, such as FUSE. On Toil-managed clusters with --enableFuse, "
+                                 "this is set to True. (default: %(default)s)")
 
     OptionType = TypeVar('OptionType')
     @classmethod
@@ -1888,4 +1900,5 @@ def setOptions(cls, setOption: OptionSetter) -> None:
         setOption("kubernetes_owner")
         setOption("kubernetes_service_account",)
         setOption("kubernetes_pod_timeout")
+        setOption("kubernetes_privileged")
 
diff --git a/src/toil/common.py b/src/toil/common.py
@@ -122,6 +122,7 @@ class Config:
     kubernetes_owner: Optional[str]
     kubernetes_service_account: Optional[str]
     kubernetes_pod_timeout: float
+    kubernetes_privileged: bool
     tes_endpoint: str
     tes_user: str
     tes_password: str
@@ -413,8 +414,6 @@ def set_option(option_name: str,
 
         self.check_configuration_consistency()
 
-        logger.debug("Loaded configuration: %s", vars(options))
-
     def check_configuration_consistency(self) -> None:
         """Old checks that cannot be fit into an action class for argparse"""
         if self.writeLogs and self.writeLogsGzip:
@@ -814,6 +813,7 @@ def __enter__(self) -> "Toil":
         set_logging_from_options(self.options)
         config = Config()
         config.setOptions(self.options)
+        logger.debug("Loaded configuration: %s", vars(self.options))
         if config.jobStore is None:
             raise RuntimeError("No jobstore provided!")
         jobStore = self.getJobStore(config.jobStore)

diff --git a/src/toil/lib/conversions.py b/src/toil/lib/conversions.py
@@ -4,7 +4,7 @@
 """
 
 import math
-from typing import SupportsInt, Tuple, Union
+from typing import SupportsInt, Tuple, Union, Optional
 
 # See https://en.wikipedia.org/wiki/Binary_prefix
 BINARY_PREFIXES = ['ki', 'mi', 'gi', 'ti', 'pi', 'ei', 'kib', 'mib', 'gib', 'tib', 'pib', 'eib']
@@ -147,3 +147,7 @@ def strtobool(val: str) -> bool:
                 return result
     raise ValueError(f"Cannot convert \"{val}\" to a bool")
 
+
+def opt_strtobool(b: Optional[str]) -> Optional[bool]:
+    """Convert an optional string representation of bool to None or bool"""
+    return b if b is None else strtobool(b)