From f0d8cf044e3a94e63fbe75fb3d242d2e0e259dd6 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Thu, 19 Sep 2024 16:49:57 +0200 Subject: [PATCH 1/9] wip --- charts/Makefile | 33 +++++++++++++++++++++------------ charts/README.md | 8 ++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/charts/Makefile b/charts/Makefile index 8bf61fee..f33c5391 100644 --- a/charts/Makefile +++ b/charts/Makefile @@ -7,33 +7,32 @@ CONFIG_DIR := $(shell dirname $(REPO_CONFIG_LOCATION)) CHART_DIRS := $(wildcard $(REPO_BASE_DIR)/charts/*/) .PHONY: .check-helmfile-installed -.check-helmfile-installed: +.check-helmfile-installed: ## Checks if helmfile is installed @if ! command -v helmfile >/dev/null 2>&1; then \ echo "'helmfile' is not installed. Install it to continue ...";\ fi -helmfile.yaml: simcore-charts/helmfile.yaml +helmfile.yaml: simcore-charts/helmfile.yaml ## Copies the helmfile.yaml to the charts directory cp $(CONFIG_DIR)/$@ $(REPO_BASE_DIR)/charts/helmfile.yaml -simcore-charts/helmfile.yaml: +simcore-charts/helmfile.yaml: ## Copies the simcore helmfile to the charts directory cp $(CONFIG_DIR)/helmfile.simcore.yaml $(REPO_BASE_DIR)/charts/$@ .PHONY: helmfile-lint -helmfile-lint: .check-helmfile-installed helmfile.yaml +helmfile-lint: .check-helmfile-installed helmfile.yaml ## Lints the helmfile set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile lint .PHONY: .helmfile-local-post-install -.helmfile-local-post-install: +.helmfile-local-post-install: ## Post install steps for local helmfile deployment @$(MAKE) -s configure-local-hosts @echo ""; @echo "Cluster has been deployed locally: http://$(MACHINE_FQDN)"; @echo " For secure connections self-signed certificates are used."; - @echo " Install their root-ca certificate in your system for smooth experience."; - @echo " For insecure connections make sure to disable automatic https redirects in your browser."; + @echo " .PHONY: helmfile-apply -helmfile-apply: .check-helmfile-installed helmfile.yaml +helmfile-apply: .check-helmfile-installed helmfile.yaml ## Applies the helmfile configuration set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply @@ -41,17 +40,27 @@ helmfile-apply: .check-helmfile-installed helmfile.yaml $(MAKE) -s .helmfile-local-post-install; \ fi +.PHONY: helmfile-sync +helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile configuration + set -a; source $(REPO_CONFIG_LOCATION); set +a; \ + helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync + + @if [ "$(MACHINE_FQDN)" = "osparc.local" ]; then \ + $(MAKE) -s .helmfile-local-post-install; \ + fi + + .PHONY: configure-local-hosts -configure-local-hosts: - @echo "Addings $(MACHINE_FQDN) hosts to /etc/hosts ..." +configure-local-hosts: ## Adds local hosts entries for the machine + @echo "Adding $(MACHINE_FQDN) hosts to /etc/hosts ..." @grep -q '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' /etc/hosts || echo '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' | sudo tee -a /etc/hosts .PHONY: helmfile-diff -helmfile-diff: .check-helmfile-installed helmfile.yaml +helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff .PHONY: helmfile-delete -helmfile-delete: .check-helmfile-installed helmfile.yaml +helmfile-delete: .check-helmfile-installed helmfile.yaml ## Deletes the helmfile configuration @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete diff --git a/charts/README.md b/charts/README.md index 13767e80..45d83630 100644 --- a/charts/README.md +++ b/charts/README.md @@ -23,6 +23,14 @@ source: https://kind.sigs.k8s.io/docs/user/quick-start Follow the instructions here: https://helm.sh/docs/intro/install/ +Install the helm-diff plugin: `helm plugin install https://github.com/databus23/helm-diff` + +`via https://doc.traefik.io/traefik/user-guides/crd-acme/#ingressroute-definition` +Install traefik-v3 CRDs: `kubectl apply -f https://raw.githubusercontent.com/traefik/traefik/v3.1/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml` + +`via https://doc.traefik.io/traefik/user-guides/crd-acme/#ingressroute-definition` +Install traefik-v3 RBAC: `kubectl apply -f https://raw.githubusercontent.com/traefik/traefik/v3.1/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml` + #### helmfile If you have a different OS / architecture, pick a different link from [release artifacts](https://github.com/helmfile/helmfile/releases) From 293f63c8c7971afeb0de64af16d01153bf76eed4 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Thu, 24 Oct 2024 10:13:28 +0200 Subject: [PATCH 2/9] Add csi-s3 and have portainer use it --- .gitignore | 3 +++ charts/csi-s3/values.yaml.gotmpl | 7 +++++++ charts/portainer/values.yaml.gotmpl | 6 ++++++ 3 files changed, 16 insertions(+) create mode 100644 charts/csi-s3/values.yaml.gotmpl diff --git a/.gitignore b/.gitignore index 0c825bcd..24edb7f6 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,6 @@ docker-compose.simcore.yml repo.config .temp .temp/** + +# By convention: `.secret` files are gitignored +**/*.secret diff --git a/charts/csi-s3/values.yaml.gotmpl b/charts/csi-s3/values.yaml.gotmpl new file mode 100644 index 00000000..7e6ff4c9 --- /dev/null +++ b/charts/csi-s3/values.yaml.gotmpl @@ -0,0 +1,7 @@ +secret: + accessKey: {{ requiredEnv "S3_ACCESS_KEY" }} + secretKey: {{ requiredEnv "S3_SECRET_KEY" }} + region: {{ requiredEnv "S3_REGION" }} + endpoint: {{ requiredEnv "S3_ENDPOINT" }} +storageClass: + singleBucket: {{ requiredEnv "S3_K8S_CSI_BUCKET_NAME" }} diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl index e89f2457..edc56479 100644 --- a/charts/portainer/values.yaml.gotmpl +++ b/charts/portainer/values.yaml.gotmpl @@ -18,6 +18,12 @@ serviceAccount: # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: portainer-sa-clusteradmin +persistence: + enabled: true + size: "10Gi" + annotations: {} + storageClass: "csi-s3" + existingClaim: podAnnotations: {} podLabels: {} From f7f72ec27e13232dababef209c92e2a3a1b983d4 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Fri, 25 Oct 2024 08:37:18 +0200 Subject: [PATCH 3/9] Change request @hrytsuk 1GB max portainer volume size --- charts/portainer/values.yaml.gotmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl index edc56479..1f5f5c44 100644 --- a/charts/portainer/values.yaml.gotmpl +++ b/charts/portainer/values.yaml.gotmpl @@ -20,7 +20,7 @@ serviceAccount: name: portainer-sa-clusteradmin persistence: enabled: true - size: "10Gi" + size: "1Gi" annotations: {} storageClass: "csi-s3" existingClaim: From 20569c78cc6d6dc7c294c294f712eef7aa7dd3c5 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Tue, 26 Nov 2024 11:39:13 +0100 Subject: [PATCH 4/9] Fix wrong filename --- ...eploy_everything_locally.sh => deploy_everything_locally.bash} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/deployments/{deploy_everything_locally.sh => deploy_everything_locally.bash} (100%) diff --git a/scripts/deployments/deploy_everything_locally.sh b/scripts/deployments/deploy_everything_locally.bash similarity index 100% rename from scripts/deployments/deploy_everything_locally.sh rename to scripts/deployments/deploy_everything_locally.bash From b2d13b7cafe1816568c9e21de049b413ee62f7c9 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 27 Nov 2024 11:38:11 +0100 Subject: [PATCH 5/9] Fix registry local deploy --- scripts/create-s3-bucket.bash | 4 ++-- services/registry/Makefile | 4 +--- services/registry/template.env | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/create-s3-bucket.bash b/scripts/create-s3-bucket.bash index 1902b1db..88313293 100755 --- a/scripts/create-s3-bucket.bash +++ b/scripts/create-s3-bucket.bash @@ -11,5 +11,5 @@ IFS=$'\n\t' docker run \ -v /etc/ssl/certs:/etc/ssl/certs:ro \ --network host \ ---env MC_HOST_local="https://${S3_ACCESS_KEY}:${S3_SECRET_KEY}@${S3_ENDPOINT}" \ -minio/mc:RELEASE.2023-06-19T19-31-19Z mb --ignore-existing local/"$1" +--env MC_HOST_local="https://${S3_ACCESS_KEY}:${S3_SECRET_KEY}@${STORAGE_DOMAIN}" \ +minio/mc:RELEASE.2023-06-19T19-31-19Z mb --insecure --ignore-existing local/"$1" diff --git a/services/registry/Makefile b/services/registry/Makefile index b0157b2e..f00ab1b3 100644 --- a/services/registry/Makefile +++ b/services/registry/Makefile @@ -24,7 +24,7 @@ endef .PHONY: up-local up-local: .init .env ${TEMP_COMPOSE}-local ## Deploys registry stack @$(create-s3-bucket) - docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME} + docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME} .PHONY: up-letsencrypt-http ## Deploys registry stack using let's encrypt http challenge up-letsencrypt-http: .init .env ${TEMP_COMPOSE}-letsencrypt-http @@ -48,8 +48,6 @@ up-master: up-dalco .PHONY: up-public ## Deploys registry on public cluster up-public: up-dalco -.PHONY: up-local ## Deploys registry on local deployment -up-local: up # Helpers ------------------------------------------------- diff --git a/services/registry/template.env b/services/registry/template.env index 550afb5a..95c9421c 100644 --- a/services/registry/template.env +++ b/services/registry/template.env @@ -10,7 +10,7 @@ REGISTRY_S3_ACCESS_KEY=${REGISTRY_S3_ACCESS_KEY} REGISTRY_S3_SECRET_KEY=${REGISTRY_S3_SECRET_KEY} S3_ACCESS_KEY=${REGISTRY_S3_ACCESS_KEY} S3_SECRET_KEY=${REGISTRY_S3_SECRET_KEY} -S3_ENDPOINT=${S3_ENDPOINT} +STORAGE_DOMAIN=${STORAGE_DOMAIN} S3_BUCKET=${REGISTRY_DOMAIN} REGISTRY_S3_BUCKET=${REGISTRY_S3_BUCKET} REGISTRY_S3_ENDPOINT=${REGISTRY_S3_ENDPOINT} From 28660ac67595b1afb2cbc3d95bd755b729bf3ded Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 27 Nov 2024 16:46:56 +0100 Subject: [PATCH 6/9] Traefik local deployment fixes --- services/traefik/docker-compose.local.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/services/traefik/docker-compose.local.yml b/services/traefik/docker-compose.local.yml index 2d8367fd..78b98f71 100644 --- a/services/traefik/docker-compose.local.yml +++ b/services/traefik/docker-compose.local.yml @@ -11,6 +11,13 @@ services: - "--api.dashboard=true" - "--log.level=${OPS_TRAEFIK_LOGLEVEL}" - "--accesslog=true" + - "--accesslog.format=json" + - "--accesslog.fields.defaultmode=keep" + - "--accesslog.fields.names.ClientUsername=keep" + - "--accesslog.fields.headers.defaultmode=keep" + - "--accesslog.fields.headers.names.User-Agent=keep" + - "--accesslog.fields.headers.names.Authorization=drop" + - "--accesslog.fields.headers.names.Content-Type=keep" - "--metrics.prometheus=true" - "--metrics.prometheus.addEntryPointsLabels=true" - "--metrics.prometheus.addServicesLabels=true" @@ -26,6 +33,9 @@ services: - "--entryPoints.https.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 - "--entryPoints.https.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 - "--entryPoints.master_postgres.address=:5432" + - "--entrypoints.http.http.redirections.entrypoint.to=https" + - "--entrypoints.http.http.redirections.entrypoint.scheme=https" + - "--entrypoints.http.http.redirections.entrypoint.permanent=true" - "--providers.swarm.endpoint=unix:///var/run/docker.sock" - "--providers.swarm.exposedByDefault=false" - "--providers.swarm.constraints=!LabelRegex(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)" @@ -45,12 +55,6 @@ services: deploy: placement: constraints: [] - labels: - # redirect http to https - - traefik.http.middlewares.http_to_https.redirectScheme.scheme=https - - traefik.http.routers.http_to_https.rule=HostRegexp(`(?P.+)`) - - traefik.http.routers.http_to_https.entrypoints=http - - traefik.http.routers.http_to_https.middlewares=http_to_https env_file: - .env configs: From 9aca16aee475c1d9ae5d38d74692cfc02a6671ba Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 27 Nov 2024 16:49:24 +0100 Subject: [PATCH 7/9] Traefik www redirect fixes --- .../jupyter_server_config.py.template | 1 + .../docker-compose.letsencrypt.dns.yml.j2 | 2 +- services/traefik/docker-compose.yml.j2 | 40 ++++++++++++------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/services/admin-panels/jupyter_server_config.py.template b/services/admin-panels/jupyter_server_config.py.template index e8722ad5..292ed0a5 100644 --- a/services/admin-panels/jupyter_server_config.py.template +++ b/services/admin-panels/jupyter_server_config.py.template @@ -1,3 +1,4 @@ +# pylint: skip-file c.ServerApp.ip = '0.0.0.0' c.ServerApp.port = 8888 c.ServerApp.open_browser = False diff --git a/services/traefik/docker-compose.letsencrypt.dns.yml.j2 b/services/traefik/docker-compose.letsencrypt.dns.yml.j2 index ff95a376..ecf0d33d 100644 --- a/services/traefik/docker-compose.letsencrypt.dns.yml.j2 +++ b/services/traefik/docker-compose.letsencrypt.dns.yml.j2 @@ -3,7 +3,7 @@ services: traefik: deploy: labels: - - traefik.http.routers.wwwsecure-catchall.tls.certresolver=myresolver + - traefik.http.routers.www-catchall.tls.certresolver=myresolver - traefik.http.routers.api.tls.certresolver=myresolver - traefik.http.middlewares.ops_whitelist_ips.ipallowlist.sourcerange=${TRAEFIK_IPWHITELIST_SOURCERANGE} # What follows is a tested workaround to ensure letsencrypt certificates for products' domains are generated diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2 index 1415674f..fce06d31 100644 --- a/services/traefik/docker-compose.yml.j2 +++ b/services/traefik/docker-compose.yml.j2 @@ -7,8 +7,15 @@ services: command: - "--api=true" - "--api.dashboard=true" + - "--accesslog=true" + - "--accesslog.format=json" + - "--accesslog.fields.defaultmode=keep" + - "--accesslog.fields.names.ClientUsername=keep" + - "--accesslog.fields.headers.defaultmode=keep" + - "--accesslog.fields.headers.names.User-Agent=keep" + - "--accesslog.fields.headers.names.Authorization=drop" + - "--accesslog.fields.headers.names.Content-Type=keep" - "--log.level=${OPS_TRAEFIK_LOGLEVEL}" - - "--accesslog=false" - "--metrics.prometheus=true" - "--metrics.prometheus.addEntryPointsLabels=true" - "--metrics.prometheus.addServicesLabels=true" @@ -18,9 +25,9 @@ services: - "--entryPoints.http.transport.respondingTimeouts.idleTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 - "--entryPoints.http.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 - "--entryPoints.http.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 - - --entrypoints.http.http.redirections.entrypoint.to=https - - --entrypoints.http.http.redirections.entrypoint.scheme=https - - --entrypoints.http.http.redirections.entrypoint.permanent=true + - "--entrypoints.http.http.redirections.entrypoint.to=https" + - "--entrypoints.http.http.redirections.entrypoint.scheme=https" + - "--entrypoints.http.http.redirections.entrypoint.permanent=true" - '--entryPoints.postgres.address=:5432' - '--entryPoints.postgres2.address=:5433' - "--entryPoints.https.address=:443" @@ -90,16 +97,15 @@ services: # via https://community.traefik.io/t/v2-2-8-global-redirect-www-to-non-www-with-http-to-https/7428 # see also: https://community.traefik.io/t/get-a-valid-ssl-certificate-for-www-domains-via-traefik-and-lets-encrypt/2023 # Global redirection: https (www.) to https - - traefik.http.routers.wwwsecure-catchall.rule=HostRegexp(`(?P(www\.).+)`) - - traefik.http.routers.wwwsecure-catchall.entrypoints=https - - traefik.http.routers.wwwsecure-catchall.tls=true - - traefik.http.routers.wwwsecure-catchall.middlewares=wwwtohttps - # middleware: http(s)://(www.) to https:// - - traefik.http.middlewares.wwwtohttps.redirectregex.regex=^https?://(?:www\.)?(.+) - - traefik.http.middlewares.wwwtohttps.redirectregex.replacement=https://$${1} - - traefik.http.middlewares.wwwtohttps.redirectregex.permanent=true + - traefik.http.routers.www-catchall.rule=Host(`www.osparc.local`) + - traefik.http.routers.www-catchall.priority=100000 + - traefik.http.routers.www-catchall.entrypoints=https,http + - traefik.http.routers.www-catchall.tls=true + - traefik.http.routers.www-catchall.middlewares=strip-www + # Explicit www domain certificate - - traefik.http.routers.wwwsecure-catchall.tls.domains[0].main=www.${MACHINE_FQDN} + - traefik.http.routers.www-catchall.tls.domains[0].main=www.${MACHINE_FQDN} # Shouldnt this be all domains?>> + ########################### # basic authentication # Note: all dollar signs in the hash need to be doubled for escaping. @@ -137,6 +143,12 @@ services: - traefik.http.middlewares.authenticated_platform_user.forwardauth.address=http://${WEBSERVER_HOST}:${WEBSERVER_PORT}/v0/auth:check - traefik.http.middlewares.authenticated_platform_user.forwardauth.trustForwardHeader=true - traefik.http.middlewares.authenticated_platform_user.forwardauth.authResponseHeaders=Set-Cookie,osparc-sc2 + # + # middleware: http(s)://(www.) to https:// + - traefik.http.middlewares.strip-www.redirectregex.regex=^(.*)www\.(.+) + - traefik.http.middlewares.strip-www.redirectregex.replacement=$${1}$${2} + - traefik.http.middlewares.strip-www.redirectregex.permanent=true + networks: public: null monitored: null @@ -145,7 +157,7 @@ services: deploy: placement: constraints: - - node.labels.ops==true + - node.labels.traefik==true labels: - traefik.enable=true - traefik.docker.network=${PUBLIC_NETWORK} From ed4a39a60a982d9b7520ccd304cad1431bfde67e Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Fri, 6 Dec 2024 14:20:09 +0100 Subject: [PATCH 8/9] Remove Explicit www TLS domain --- services/traefik/docker-compose.yml.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2 index fce06d31..98cf7ea6 100644 --- a/services/traefik/docker-compose.yml.j2 +++ b/services/traefik/docker-compose.yml.j2 @@ -103,9 +103,6 @@ services: - traefik.http.routers.www-catchall.tls=true - traefik.http.routers.www-catchall.middlewares=strip-www - # Explicit www domain certificate - - traefik.http.routers.www-catchall.tls.domains[0].main=www.${MACHINE_FQDN} # Shouldnt this be all domains?>> - ########################### # basic authentication # Note: all dollar signs in the hash need to be doubled for escaping. From 97f67e35ebd2460b2e1c35540f7cb8407c8ddd38 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Fri, 6 Dec 2024 15:19:59 +0100 Subject: [PATCH 9/9] Enhance www redirect regex @hrytsuk --- services/traefik/docker-compose.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2 index 98cf7ea6..71e6bb30 100644 --- a/services/traefik/docker-compose.yml.j2 +++ b/services/traefik/docker-compose.yml.j2 @@ -142,8 +142,8 @@ services: - traefik.http.middlewares.authenticated_platform_user.forwardauth.authResponseHeaders=Set-Cookie,osparc-sc2 # # middleware: http(s)://(www.) to https:// - - traefik.http.middlewares.strip-www.redirectregex.regex=^(.*)www\.(.+) - - traefik.http.middlewares.strip-www.redirectregex.replacement=$${1}$${2} + - traefik.http.middlewares.strip-www.redirectregex.regex=^(https?)://www\.(.+) + - traefik.http.middlewares.strip-www.redirectregex.replacement=$${1}://$${2} - traefik.http.middlewares.strip-www.redirectregex.permanent=true networks: