From 9e3cf66f5bb2a287899af589616c3dd06e64184e Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 8 Mar 2022 11:37:00 -0500 Subject: [PATCH] Removing custom dask_gateway from qhub Closes #1012 --- .github/workflows/image.yaml | 1 - docs/source/installation/configuration.md | 7 +-- docs/source/installation/existing.md | 2 - qhub/initialize.py | 1 - qhub/schema.py | 1 - qhub/stages/input_vars.py | 3 -- qhub/template/image/Dockerfile.dask-gateway | 50 ------------------- qhub/template/image/docker-bake.json | 10 ---- .../07-kubernetes-services/dask_gateway.tf | 10 ---- .../dask-gateway/files/controller_config.py | 35 ------------- qhub/upgrade.py | 4 ++ 11 files changed, 6 insertions(+), 118 deletions(-) delete mode 100644 qhub/template/image/Dockerfile.dask-gateway diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 7cf9a4ae3..657afcfdb 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -27,7 +27,6 @@ jobs: - jupyterlab - jupyterhub - dask-worker - - dask-gateway steps: - name: 'Checkout Infrastructure' uses: actions/checkout@main diff --git a/docs/source/installation/configuration.md b/docs/source/installation/configuration.md index be3e878af..246d0b587 100644 --- a/docs/source/installation/configuration.md +++ b/docs/source/installation/configuration.md @@ -502,16 +502,14 @@ terraform_state: Default images are to the default image run if not specified in a profile (described in the next section). The `jupyterhub` key controls the jupyterhub image run. These control the docker image used to run -JupyterHub, the default JupyterLab image, the default Dask worker -image, and Dask Gateway docker image. +JupyterHub, the default JupyterLab image, and the default Dask worker +image. ```yaml default_images: jupyterhub: "quansight/qhub-jupyterhub:v||QHUB_VERSION||" jupyterlab: "quansight/qhub-jupyterlab:v||QHUB_VERSION||" dask_worker: "quansight/qhub-dask-worker:v||QHUB_VERSION||" - dask_gateway: "quansight/qhub-dask-gateway:v||QHUB_VERSION||" - conda_store: "quansight/conda-store-server:v0.3.3" ``` ## Storage @@ -831,7 +829,6 @@ default_images: jupyterhub: "quansight/qhub-jupyterhub:v||QHUB_VERSION||" jupyterlab: "quansight/qhub-jupyterlab:v||QHUB_VERSION||" dask_worker: "quansight/qhub-dask-worker:v||QHUB_VERSION||" - conda_store: "quansight/conda-store-server:v0.3.3" theme: jupyterhub: diff --git a/docs/source/installation/existing.md b/docs/source/installation/existing.md index f146b04b8..a1ada21ba 100644 --- a/docs/source/installation/existing.md +++ b/docs/source/installation/existing.md @@ -128,8 +128,6 @@ default_images: jupyterhub: quansight/qhub-jupyterhub:v0.3.13 jupyterlab: quansight/qhub-jupyterlab:v0.3.13 dask_worker: quansight/qhub-dask-worker:v0.3.13 - dask_gateway: quansight/qhub-dask-gateway:v0.3.13 - conda_store: quansight/conda-store-server:v0.3.3 storage: conda_store: 60Gi shared_filesystem: 100Gi diff --git a/qhub/initialize.py b/qhub/initialize.py index ed505769c..1b2744c06 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -37,7 +37,6 @@ "jupyterhub": f"quansight/qhub-jupyterhub:{qhub_image_tag}", "jupyterlab": f"quansight/qhub-jupyterlab:{qhub_image_tag}", "dask_worker": f"quansight/qhub-dask-worker:{qhub_image_tag}", - "dask_gateway": f"quansight/qhub-dask-gateway:{qhub_image_tag}", }, "storage": {"conda_store": "60Gi", "shared_filesystem": "100Gi"}, "theme": { diff --git a/qhub/schema.py b/qhub/schema.py index d9c8fceea..29c673e31 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -120,7 +120,6 @@ class DefaultImages(Base): jupyterhub: str jupyterlab: str dask_worker: str - dask_gateway: str # =========== Authentication ============== diff --git a/qhub/stages/input_vars.py b/qhub/stages/input_vars.py index d50701918..f0c28808a 100644 --- a/qhub/stages/input_vars.py +++ b/qhub/stages/input_vars.py @@ -238,9 +238,6 @@ def stage_07_kubernetes_services(stage_outputs, config): .get("extraEnv", []) ), # dask-gateway - "dask-gateway-image": _split_docker_image_name( - config["default_images"]["dask_gateway"] - ), "dask-worker-image": _split_docker_image_name( config["default_images"]["dask_worker"] ), diff --git a/qhub/template/image/Dockerfile.dask-gateway b/qhub/template/image/Dockerfile.dask-gateway deleted file mode 100644 index 37551743f..000000000 --- a/qhub/template/image/Dockerfile.dask-gateway +++ /dev/null @@ -1,50 +0,0 @@ -FROM python:3.8.6-slim-buster as dependencies -LABEL MAINTAINER="Quansight" - -RUN apt-get update \ - && apt-get install -y tini wget \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir \ - aiohttp==3.7.2 \ - colorlog \ - cryptography \ - traitlets==4.3.3 \ - pyyaml \ - kubernetes-asyncio==11.0.0 - -# Build dask-gateway-server from source in a builder stage -FROM dependencies AS builder - -RUN mkdir -p /tmp/workdir -RUN mkdir -p /tmp/install-prefix - -RUN cd /tmp && \ - wget -O dask-gateway.tar.gz "https://github.com/costrouc/dask-gateway/archive/refs/heads/kubernetes-controller-callables.tar.gz" && \ - tar -xf dask-gateway.tar.gz && \ - ls -la && \ - cd dask-gateway-kubernetes-controller-callables/dask-gateway-server && \ - cp -r . /tmp/workdir && \ - rm -rf /tmp/dask-gateway-kubernetes-controller-callables /tmp/dask-gateway.tar.gz - -WORKDIR /tmp/workdir/ -RUN python setup.py install \ - --no-build-proxy \ - --single-version-externally-managed \ - --record=record.txt \ - --prefix /tmp/install-prefix - -# Final image - merge dependencies and built dask-gateway -FROM dependencies - -COPY --from=builder /tmp/install-prefix/bin/dask-gateway-server /usr/local/bin/ -COPY --from=builder /tmp/install-prefix/lib /usr/local/lib/ - -# Create non-root user and working directory -WORKDIR /srv/dask-gateway -RUN useradd -m -U -u 1000 dask && chown dask:dask /srv/dask-gateway -USER 1000:1000 - -ENTRYPOINT ["tini", "-g", "--"] -CMD ["dask-gateway-server", "--config", "/etc/dask-gateway/dask_gateway_config.py"] diff --git a/qhub/template/image/docker-bake.json b/qhub/template/image/docker-bake.json index 55f0d4198..8590dcd32 100644 --- a/qhub/template/image/docker-bake.json +++ b/qhub/template/image/docker-bake.json @@ -7,7 +7,6 @@ "group": { "default": { "targets": [ - "dask-gateway", "dask-worker", "jupyterhub", "jupyterlab" @@ -15,15 +14,6 @@ } }, "target": { - "dask-gateway": { - "context": "./", - "dockerfile": "Dockerfile.dask-gateway", - "tags": [ - "docker.io/quansight/qhub-dask-gateway:${TAG}", - "ghcr.io/quansight/qhub-dask-gateway:${TAG}", - "quay.io/quansight/qhub-dask-gateway:${TAG}" - ] - }, "dask-worker": { "context": "./", "dockerfile": "Dockerfile.dask-worker", diff --git a/qhub/template/stages/07-kubernetes-services/dask_gateway.tf b/qhub/template/stages/07-kubernetes-services/dask_gateway.tf index f2057f4cd..7fc7ee3d4 100644 --- a/qhub/template/stages/07-kubernetes-services/dask_gateway.tf +++ b/qhub/template/stages/07-kubernetes-services/dask_gateway.tf @@ -7,14 +7,6 @@ variable "dask-worker-image" { }) } -variable "dask-gateway-image" { - description = "Dask worker image" - type = object({ - name = string - tag = string - }) -} - variable "dask-gateway-profiles" { description = "Dask Gateway profiles to expose to user" default = [] @@ -32,8 +24,6 @@ module "dask-gateway" { external-url = var.endpoint cluster-image = var.dask-worker-image - gateway-image = var.dask-gateway-image - controller-image = var.dask-gateway-image general-node-group = var.node_groups.general worker-node-group = var.node_groups.worker diff --git a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py index 16900fcba..245f4686f 100644 --- a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py +++ b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py @@ -32,38 +32,3 @@ def dask_gateway_config(path="/var/lib/dask-gateway/config.json"): c.KubeController.proxy_web_entrypoint = "websecure" c.KubeController.proxy_tcp_entrypoint = "tcp" - - -# override default make_ingressroute to add tls cert resolver -def make_ingressroute(self, cluster_name, namespace): - route = f"{self.proxy_prefix}/clusters/{namespace}.{cluster_name}/" - return { - "apiVersion": "traefik.containo.us/v1alpha1", - "kind": "IngressRoute", - "metadata": { - "labels": self.get_labels(cluster_name, "dask-scheduler"), - "annotations": self.common_annotations, - "name": f"dask-{cluster_name}", - }, - "spec": { - "entryPoints": [self.proxy_web_entrypoint], - "routes": [ - { - "kind": "Rule", - "match": f"PathPrefix(`{route}`)", - "services": [ - { - "name": self.make_service_name(cluster_name), - "namespace": namespace, - "port": 8787, - } - ], - "middlewares": self.proxy_web_middlewares, - } - ], - "tls": {}, - }, - } - - -c.KubeController.make_ingressroute = make_ingressroute diff --git a/qhub/upgrade.py b/qhub/upgrade.py index 7c9de1810..7a6bc777d 100644 --- a/qhub/upgrade.py +++ b/qhub/upgrade.py @@ -318,6 +318,10 @@ def _version_specific_upgrade( if "conda_store" in config["default_images"]: del config["default_images"]["conda_store"] + # Remove dask_gateway image from default_images + if "dask_gateway" in config["default_images"]: + del config["default_images"]["dask_gateway"] + # Create root password default_password = "".join( secrets.choice(string.ascii_letters + string.digits) for i in range(16)