From bc0f85ef21fe1419e8ce8d1518b881d427068028 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 8 Mar 2022 11:37:00 -0500 Subject: [PATCH 1/3] Removing custom dask_gateway from qhub Closes #1012 --- .github/workflows/image.yaml | 1 - docs/source/installation/configuration.md | 5 +- docs/source/installation/existing.md | 2 - qhub/initialize.py | 1 - qhub/schema.py | 1 - qhub/stages/input_vars.py | 3 -- qhub/template/image/Dockerfile.dask-gateway | 50 ------------------- qhub/template/image/docker-bake.json | 10 ---- .../07-kubernetes-services/dask_gateway.tf | 10 ---- .../dask-gateway/files/controller_config.py | 35 ------------- qhub/upgrade.py | 4 ++ 11 files changed, 6 insertions(+), 116 deletions(-) delete mode 100644 qhub/template/image/Dockerfile.dask-gateway diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 7cf9a4ae3d..657afcfdbc 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -27,7 +27,6 @@ jobs: - jupyterlab - jupyterhub - dask-worker - - dask-gateway steps: - name: 'Checkout Infrastructure' uses: actions/checkout@main diff --git a/docs/source/installation/configuration.md b/docs/source/installation/configuration.md index e117b07709..9e559a5a06 100644 --- a/docs/source/installation/configuration.md +++ b/docs/source/installation/configuration.md @@ -495,15 +495,14 @@ terraform_state: Default images are to the default image run if not specified in a profile (described in the next section). The `jupyterhub` key controls the jupyterhub image run. These control the docker image used to run -JupyterHub, the default JupyterLab image, the default Dask worker -image, and Dask Gateway docker image. +JupyterHub, the default JupyterLab image, and the default Dask worker +image. ```yaml default_images: jupyterhub: "quansight/qhub-jupyterhub:v||QHUB_VERSION||" jupyterlab: "quansight/qhub-jupyterlab:v||QHUB_VERSION||" dask_worker: "quansight/qhub-dask-worker:v||QHUB_VERSION||" - dask_gateway: "quansight/qhub-dask-gateway:v||QHUB_VERSION||" ``` ## Storage diff --git a/docs/source/installation/existing.md b/docs/source/installation/existing.md index f146b04b82..a1ada21ba4 100644 --- a/docs/source/installation/existing.md +++ b/docs/source/installation/existing.md @@ -128,8 +128,6 @@ default_images: jupyterhub: quansight/qhub-jupyterhub:v0.3.13 jupyterlab: quansight/qhub-jupyterlab:v0.3.13 dask_worker: quansight/qhub-dask-worker:v0.3.13 - dask_gateway: quansight/qhub-dask-gateway:v0.3.13 - conda_store: quansight/conda-store-server:v0.3.3 storage: conda_store: 60Gi shared_filesystem: 100Gi diff --git a/qhub/initialize.py b/qhub/initialize.py index f35dc30397..8db10d0fd1 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -37,7 +37,6 @@ "jupyterhub": f"quansight/qhub-jupyterhub:{qhub_image_tag}", "jupyterlab": f"quansight/qhub-jupyterlab:{qhub_image_tag}", "dask_worker": f"quansight/qhub-dask-worker:{qhub_image_tag}", - "dask_gateway": f"quansight/qhub-dask-gateway:{qhub_image_tag}", }, "storage": {"conda_store": "60Gi", "shared_filesystem": "100Gi"}, "theme": { diff --git a/qhub/schema.py b/qhub/schema.py index 074968d2a3..a4ba7d4969 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -121,7 +121,6 @@ class DefaultImages(Base): jupyterhub: str jupyterlab: str dask_worker: str - dask_gateway: str # =========== Authentication ============== diff --git a/qhub/stages/input_vars.py b/qhub/stages/input_vars.py index 6f4e5f94b5..80c6515a3f 100644 --- a/qhub/stages/input_vars.py +++ b/qhub/stages/input_vars.py @@ -244,9 +244,6 @@ def stage_07_kubernetes_services(stage_outputs, config): .get("extraEnv", []) ), # dask-gateway - "dask-gateway-image": _split_docker_image_name( - config["default_images"]["dask_gateway"] - ), "dask-worker-image": _split_docker_image_name( config["default_images"]["dask_worker"] ), diff --git a/qhub/template/image/Dockerfile.dask-gateway b/qhub/template/image/Dockerfile.dask-gateway deleted file mode 100644 index 37551743f4..0000000000 --- a/qhub/template/image/Dockerfile.dask-gateway +++ /dev/null @@ -1,50 +0,0 @@ -FROM python:3.8.6-slim-buster as dependencies -LABEL MAINTAINER="Quansight" - -RUN apt-get update \ - && apt-get install -y tini wget \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir \ - aiohttp==3.7.2 \ - colorlog \ - cryptography \ - traitlets==4.3.3 \ - pyyaml \ - kubernetes-asyncio==11.0.0 - -# Build dask-gateway-server from source in a builder stage -FROM dependencies AS builder - -RUN mkdir -p /tmp/workdir -RUN mkdir -p /tmp/install-prefix - -RUN cd /tmp && \ - wget -O dask-gateway.tar.gz "https://github.com/costrouc/dask-gateway/archive/refs/heads/kubernetes-controller-callables.tar.gz" && \ - tar -xf dask-gateway.tar.gz && \ - ls -la && \ - cd dask-gateway-kubernetes-controller-callables/dask-gateway-server && \ - cp -r . /tmp/workdir && \ - rm -rf /tmp/dask-gateway-kubernetes-controller-callables /tmp/dask-gateway.tar.gz - -WORKDIR /tmp/workdir/ -RUN python setup.py install \ - --no-build-proxy \ - --single-version-externally-managed \ - --record=record.txt \ - --prefix /tmp/install-prefix - -# Final image - merge dependencies and built dask-gateway -FROM dependencies - -COPY --from=builder /tmp/install-prefix/bin/dask-gateway-server /usr/local/bin/ -COPY --from=builder /tmp/install-prefix/lib /usr/local/lib/ - -# Create non-root user and working directory -WORKDIR /srv/dask-gateway -RUN useradd -m -U -u 1000 dask && chown dask:dask /srv/dask-gateway -USER 1000:1000 - -ENTRYPOINT ["tini", "-g", "--"] -CMD ["dask-gateway-server", "--config", "/etc/dask-gateway/dask_gateway_config.py"] diff --git a/qhub/template/image/docker-bake.json b/qhub/template/image/docker-bake.json index 55f0d4198a..8590dcd32a 100644 --- a/qhub/template/image/docker-bake.json +++ b/qhub/template/image/docker-bake.json @@ -7,7 +7,6 @@ "group": { "default": { "targets": [ - "dask-gateway", "dask-worker", "jupyterhub", "jupyterlab" @@ -15,15 +14,6 @@ } }, "target": { - "dask-gateway": { - "context": "./", - "dockerfile": "Dockerfile.dask-gateway", - "tags": [ - "docker.io/quansight/qhub-dask-gateway:${TAG}", - "ghcr.io/quansight/qhub-dask-gateway:${TAG}", - "quay.io/quansight/qhub-dask-gateway:${TAG}" - ] - }, "dask-worker": { "context": "./", "dockerfile": "Dockerfile.dask-worker", diff --git a/qhub/template/stages/07-kubernetes-services/dask_gateway.tf b/qhub/template/stages/07-kubernetes-services/dask_gateway.tf index f2057f4cd0..7fc7ee3d42 100644 --- a/qhub/template/stages/07-kubernetes-services/dask_gateway.tf +++ b/qhub/template/stages/07-kubernetes-services/dask_gateway.tf @@ -7,14 +7,6 @@ variable "dask-worker-image" { }) } -variable "dask-gateway-image" { - description = "Dask worker image" - type = object({ - name = string - tag = string - }) -} - variable "dask-gateway-profiles" { description = "Dask Gateway profiles to expose to user" default = [] @@ -32,8 +24,6 @@ module "dask-gateway" { external-url = var.endpoint cluster-image = var.dask-worker-image - gateway-image = var.dask-gateway-image - controller-image = var.dask-gateway-image general-node-group = var.node_groups.general worker-node-group = var.node_groups.worker diff --git a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py index 16900fcba2..245f4686fa 100644 --- a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py +++ b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/controller_config.py @@ -32,38 +32,3 @@ def dask_gateway_config(path="/var/lib/dask-gateway/config.json"): c.KubeController.proxy_web_entrypoint = "websecure" c.KubeController.proxy_tcp_entrypoint = "tcp" - - -# override default make_ingressroute to add tls cert resolver -def make_ingressroute(self, cluster_name, namespace): - route = f"{self.proxy_prefix}/clusters/{namespace}.{cluster_name}/" - return { - "apiVersion": "traefik.containo.us/v1alpha1", - "kind": "IngressRoute", - "metadata": { - "labels": self.get_labels(cluster_name, "dask-scheduler"), - "annotations": self.common_annotations, - "name": f"dask-{cluster_name}", - }, - "spec": { - "entryPoints": [self.proxy_web_entrypoint], - "routes": [ - { - "kind": "Rule", - "match": f"PathPrefix(`{route}`)", - "services": [ - { - "name": self.make_service_name(cluster_name), - "namespace": namespace, - "port": 8787, - } - ], - "middlewares": self.proxy_web_middlewares, - } - ], - "tls": {}, - }, - } - - -c.KubeController.make_ingressroute = make_ingressroute diff --git a/qhub/upgrade.py b/qhub/upgrade.py index 7c9de18102..7a6bc777de 100644 --- a/qhub/upgrade.py +++ b/qhub/upgrade.py @@ -318,6 +318,10 @@ def _version_specific_upgrade( if "conda_store" in config["default_images"]: del config["default_images"]["conda_store"] + # Remove dask_gateway image from default_images + if "dask_gateway" in config["default_images"]: + del config["default_images"]["dask_gateway"] + # Create root password default_password = "".join( secrets.choice(string.ascii_letters + string.digits) for i in range(16) From c240f67cfc4aefdd8bdf054638c651963b62f570 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Tue, 8 Mar 2022 15:11:15 -0500 Subject: [PATCH 2/3] Adding idle timeout to dask gateway --- .../kubernetes/services/dask-gateway/files/gateway_config.py | 1 + .../modules/kubernetes/services/dask-gateway/variables.tf | 2 ++ 2 files changed, 3 insertions(+) diff --git a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/gateway_config.py index 75f702e4dc..271436662d 100644 --- a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/gateway_config.py +++ b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/gateway_config.py @@ -30,6 +30,7 @@ def dask_gateway_config(path="/var/lib/dask-gateway/config.json"): ) c.KubeClusterConfig.image_pull_policy = config["cluster"]["image_pull_policy"] c.KubeClusterConfig.environment = config["cluster"]["environment"] +c.KubeClusterConfig.idle_timeout = config["cluster"]["idle_timeout"] c.KubeClusterConfig.scheduler_cores = config["cluster"]["scheduler_cores"] c.KubeClusterConfig.scheduler_cores_limit = config["cluster"]["scheduler_cores_limit"] diff --git a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/variables.tf b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/variables.tf index 9fb5bf7323..1e3b1f2bee 100644 --- a/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/variables.tf +++ b/qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/variables.tf @@ -144,6 +144,7 @@ variable "cluster" { worker_extra_container_config = any worker_extra_pod_config = any # additional fields + idle_timeout = number image_pull_policy = string environment = map(string) }) @@ -163,6 +164,7 @@ variable "cluster" { worker_extra_container_config = {} worker_extra_pod_config = {} # additional fields + idle_timeout = 1800 # 30 minutes image_pull_policy = "IfNotPresent" environment = {} } From 3ae303cbacc1fc1109d7d0582c8830703293aa91 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 18 Mar 2022 14:27:18 -0400 Subject: [PATCH 3/3] Fixing broken links --- docs/source/installation/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/installation/configuration.md b/docs/source/installation/configuration.md index 9e559a5a06..882db4b475 100644 --- a/docs/source/installation/configuration.md +++ b/docs/source/installation/configuration.md @@ -450,7 +450,7 @@ local: ## Terraform state Terraform manages the state of all the deployed resources via -[backends](https://www.terraform.io/docs/language/settings/backends/index.html). Terraform +[backends](https://www.terraform.io/language/settings/backends). Terraform requires storing the state in order to keep track of the names, ids, and states of deployed resources. The simplest approach is storing the state on the local filesystem but isn't recommended and isn't the @@ -459,7 +459,7 @@ default of QHub. `terraform_state` is either `remote`, `existing` or control the state of the cluster `local` via tfstate file (not recommended), on an already `existing` terraform state store or remotely and auto creating the terraform state store. See [terraform -remote state](https://www.terraform.io/docs/language/state/index.html) +remote state](https://www.terraform.io/language/state/remote) docs. If you are doing anything other than testing we highly recommend `remote` unless you know what you are doing.