[ws-daemon] Fold ws-manager-node into ws-daemon

olearycrew · Oct 19, 2020 · 7172d82 · 7172d82
1 parent 6d93dd8
commit 7172d82
Show file tree

Hide file tree

Showing 119 changed files with 2,702 additions and 7,246 deletions.
diff --git a/.werft/build.js b/.werft/build.js
@@ -110,8 +110,8 @@ async function deployToDev(version, previewWithHttps, workspaceFeatureFlags) {
     const namespace = `staging-${destname}`;
     const domain = `${destname}.staging.gitpod-dev.com`;
     const url = `${!!previewWithHttps ? "https" : "http"}://${domain}`;
-    const wssyncPort = `1${Math.floor(Math.random()*1000)}`;
-    const wsmanNodePort = `2${Math.floor(Math.random()*1000)}`;
+    const wsdaemonPort = `1${Math.floor(Math.random()*1000)}`;
+    const registryProxyPort = `2${Math.floor(Math.random()*1000)}`;
     const registryNodePort = `${30000 + Math.floor(Math.random()*1000)}`;
 
     try {
@@ -168,7 +168,7 @@ async function deployToDev(version, previewWithHttps, workspaceFeatureFlags) {
         exec(`/usr/local/bin/helm3 delete jaeger-${destname} || echo jaeger-${destname} was not installed yet`, {slice: 'predeploy cleanup'});
 
         let objs = [];
-        ["ws-scheduler", "node-daemon", "cluster", "workspace", "jaeger", "jaeger-agent", "ws-sync", "ws-manager-node"].forEach(comp => 
+        ["ws-scheduler", "node-daemon", "cluster", "workspace", "jaeger", "jaeger-agent", "ws-sync", "ws-manager-node", "ws-daemon"].forEach(comp => 
             ["ClusterRole", "ClusterRoleBinding", "PodSecurityPolicy"].forEach(kind =>
                 shell
                     .exec(`kubectl get ${kind} -l component=${comp} --no-headers -o=custom-columns=:metadata.name | grep ${namespace}-ns`)
@@ -203,8 +203,8 @@ async function deployToDev(version, previewWithHttps, workspaceFeatureFlags) {
     flags+=` --set version=${version}`;
     flags+=` --set hostname=${domain}`;
     flags+=` --set devBranch=${destname}`;
-    flags+=` --set components.wsSync.servicePort=${wssyncPort}`;
-    flags+=` --set components.wsManagerNode.registryProxyPort=${wsmanNodePort}`;
+    flags+=` --set components.wsDaemon.servicePort=${wsdaemonPort}`;
+    flags+=` --set components.wsDaemon.registryProxyPort=${registryProxyPort}`;
     flags+=` --set ingressMode=${context.Annotations.ingressMode || "hosts"}`;
     workspaceFeatureFlags.forEach((f, i) => {
         flags+=` --set components.server.defaultFeatureFlags[${i}]='${f}'`

diff --git a/chart/templates/image-builder-deployment.yaml b/chart/templates/image-builder-deployment.yaml
@@ -3,7 +3,6 @@
 
 {{ $comp := .Values.components.imageBuilder -}}
 {{- $this := dict "root" . "gp" $.Values "comp" $comp -}}
-{{- $thisWsSync := dict "root" . "comp" .Values.components.wsSync -}}
 {{- if not $comp.disabled -}}
 apiVersion: apps/v1
 kind: Deployment

diff --git a/chart/templates/server-deployment.yaml b/chart/templates/server-deployment.yaml
@@ -145,7 +145,7 @@ spec:
     {{- end }}
         - name: SESSION_SECRET
           value: {{ $comp.sessionSecret | quote }}
-    {{- if eq .Values.components.wsSync.remoteStorage.kind "minio" }}
+    {{- if eq .Values.components.wsDaemon.remoteStorage.kind "minio" }}
         - name: GITPOD_STORAGE_CLIENT
           value: minio
         - name: MINIO_END_POINT
@@ -195,13 +195,13 @@ spec:
             secretKeyRef:
               name: server-proxy-apikey
               key: apikey
-{{- if (or .Values.components.wsSync.remoteStorage.gcloud $comp.storage) }}
+{{- if (or .Values.components.wsDaemon.remoteStorage.gcloud $comp.storage) }}
         - name: GCLOUD_PROJECT_ID
-          value: {{ .Values.components.wsSync.remoteStorage.gcloud.projectId }}
+          value: {{ .Values.components.wsDaemon.remoteStorage.gcloud.projectId }}
         - name: GCLOUD_REGION
-          value: {{ .Values.components.wsSync.remoteStorage.gcloud.region }}
+          value: {{ .Values.components.wsDaemon.remoteStorage.gcloud.region }}
         - name: GCLOUD_CREDENTIALS_FILE
-          value: {{ base .Values.components.wsSync.remoteStorage.gcloud.credentialsFile }}
+          value: {{ base .Values.components.wsDaemon.remoteStorage.gcloud.credentialsFile }}
 {{- end }}
         - name: GITPOD_GARBAGE_COLLECTION_DISABLED
           value: {{ $comp.garbageCollection.disabled | default "false" | quote }}

diff --git a/chart/templates/ws-daemon-clusterrole.yaml b/chart/templates/ws-daemon-clusterrole.yaml
@@ -18,4 +18,22 @@ rules:
   verbs:
   - "use"
   resourceNames:
-  - "{{ .Release.Namespace }}-ns-privileged-unconfined"
+  - "{{ .Release.Namespace }}-ns-privileged-unconfined"
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - services
+  verbs:
+  - get
+  - list
+  - watch
diff --git a/chart/templates/ws-daemon-configmap.yaml b/chart/templates/ws-daemon-configmap.yaml
@@ -1,7 +1,97 @@
 # Copyright (c) 2020 TypeFox GmbH. All rights reserved.
 # Licensed under the MIT License. See License-MIT.txt in the project root for license information.
 
-{{ $comp := .Values.components.wsSync -}}
+{{- define "ws-daemon.config" }}
+{{ $comp := .comp -}}
+{{ with .root }}
+daemon:
+  runtime:
+    namespace: {{ .Release.Namespace | quote }}
+    containerRuntime:
+      runtime: "containerd"
+      containerd:
+        socket: "/mnt/containerd.sock"
+      nodeToContainerMapping:
+        {{- range $idx, $pth := $comp.containerRuntime.nodeRoots }}
+        {{ $pth | quote }}: "/mnt/node{{ $idx }}"
+        {{- end }}
+      mounts:
+        proc: "/mnt/mounts"
+  content:
+    workingArea: "/mnt/workingarea"
+    {{- if (and $comp.workspaceSizeLimit (not (eq $comp.workspaceSizeLimit ""))) }}
+    workspaceSizeLimit: {{ ($comp.workspaceSizeLimit | default "0g") | quote }}
+    {{- end }}
+    storage:
+{{ toYaml $comp.remoteStorage | indent 6 }}
+    backup:
+      timeout: "5m"
+      attempts: 3
+    fullWorkspaceBackup:
+      workdir: "/mnt/node0/gitpod-{{ .Release.Namespace }}"
+  uidmapper:
+    procLocation: "/proc"
+    rootUIDRange:
+      start: 33333
+      size: 1
+    userUIDRange:
+      - start: 100000
+        size: 70000
+  resources:
+    cgroupBasePath: "/mnt/node-cgroups"
+    # We don't split our actual budget equally amongst participants. Instead we assume we have a maximum
+    # number of over-consumers. We hand out CPU in buckets:
+    #   three minutes of 5 CPUs: 5 [numCPU] * 100 [jiffies/sec] * (3 * 60) [seconds] = 90000
+    #   five minutes  of 4 CPUs: 4 [numCPU] * 100 [jiffies/sec] * (5 * 60) [seconds] = 120000
+    #   remainder of 2 CPUs where a user has to stay below sustained use of 1.8 CPUs for 5 minutes:
+    #                          1.8 [numCPU] * 100 [jiffies/sec] * (5 * 60) [seconds] = 54000
+    # 
+    # Compared to the actual budget, this is severe overbooking:
+    # budget is (numCPU * overbookingFactor * jiffies/sec * controlPeriod[sec]) / numWS = [jiffies / workspace]
+    #           (16     * 1                 * 100         * (15 * 60))          / 25    = 57600
+    #
+    # We express everything in jiffies/sec where 1 jiffie is 1% of a CPU core.
+    cpuBuckets: 
+      - budget: 90000
+        limit: 500
+      - budget: 120000
+        limit: 400
+      - budget: 54000
+        limit: 200
+    processPriorities:
+      supervisor: 0
+      theia: 5
+      shell: 6
+      default: 10
+    controlPeriod: "15m"
+    samplingPeriod: "10s"
+  hosts:
+    enabled: true
+    nodeHostsFile: "/mnt/hosts"
+    serviceProxy:
+      enabled: true
+      mapping:
+        - selector: "component=registry-facade,feature=registry"
+          alias: {{ (printf "reg.%s" (.Values.components.registryFacade.hostname | default .Values.hostname)) | quote }}
+          proxyPort: {{ $comp.registryProxyPort }}
+  disk:
+    path: "/mnt/wsdaemon-workingarea"
+    minBytesAvail: 21474836480
+service:
+  address: ":{{ $comp.servicePort }}"
+  tls:
+    ca: "/certs/ca.crt"
+    crt: "/certs/tls.crt"
+    key: "/certs/tls.key"
+prometheus:
+  address: ":9500"
+pprof:
+  address: ":6060"
+{{ end }}
+{{ end }}
+
+{{ $comp := .Values.components.wsDaemon -}}
+{{- $this := dict "root" . "gp" $.Values "comp" $comp -}}
 {{- if not $comp.disabled -}}
 apiVersion: v1
 kind: ConfigMap
@@ -14,50 +104,5 @@ metadata:
     heritage: "{{ .Release.Service }}"
 data:
   config.json: |-
-    {
-        "syncd": {
-            "workingArea": "/mnt/workingarea",
-            "backup": {
-                "timeout": "5m",
-                "attempts": 3
-            },
-            {{- if (and $comp.workspaceSizeLimit (not (eq $comp.workspaceSizeLimit ""))) }}
-            "workspaceSizeLimit": {{ ($comp.workspaceSizeLimit | default "0g") | quote }},
-            {{- end }}
-            "storage": {{ toJson $comp.remoteStorage }},
-            "fullWorkspaceBackup": {
-                "enabled": {{ $comp.fullWorkspaceBackup.enabled }},
-                "workdir": "/mnt/node0/gitpod-{{ .Release.Namespace }}",
-                "cri": {
-                    "nodeToContainerMapping": {
-                        {{- range $idx, $pth := $comp.fullWorkspaceBackup.nodeRoots }}
-                        {{ if gt $idx 0 }},{{ end -}}{{ $pth | quote }}: "/mnt/node{{ $idx }}"
-                        {{- end }}
-                    },
-                    "mounts": {
-                        "proc": "/mnt/mounts"
-                    },
-                    "runtime": "containerd",
-                    "containerd": {
-                        "socket": "/mnt/containerd.sock"
-                    }
-                }
-            },
-            "namespace": "{{ .Release.Namespace }}"
-        },
-        "service": {
-            "address": ":{{ $comp.servicePort }}",
-            "tls": {
-                "ca": "/certs/ca.crt",
-                "crt": "/certs/tls.crt",
-                "key": "/certs/tls.key"
-            }
-        },
-        "prometheus": {
-            "address": ":9500"
-        },
-        "pprof": {
-            "address": "localhost:6060"
-        }
-    }
-{{- end -}}
+{{ (include "ws-daemon.config" $this) | fromYaml | toPrettyJson | indent 4 }}
+{{- end }}
diff --git a/chart/templates/ws-daemon-daemonset.yaml b/chart/templates/ws-daemon-daemonset.yaml
@@ -1,7 +1,7 @@
 # Copyright (c) 2020 TypeFox GmbH. All rights reserved.
 # Licensed under the MIT License. See License-MIT.txt in the project root for license information.
 
-{{ $comp := .Values.components.wsSync -}}
+{{ $comp := .Values.components.wsDaemon -}}
 {{- $this := dict "root" . "gp" $.Values "comp" $comp -}}
 {{- if not $comp.disabled -}}
 apiVersion: apps/v1
@@ -53,6 +53,7 @@ spec:
         operator: "Exists"
         effect: "NoExecute"
       serviceAccountName: ws-daemon
+      hostPID: true
       volumes:
       - hostPath:
           path: {{ $comp.hostWorkspaceArea }}
@@ -64,12 +65,11 @@ spec:
       - name: config
         configMap:
           name: {{ template "gitpod.comp.configMap" $this }}
-      {{- if $comp.fullWorkspaceBackup.enabled }}
       - name: containerd-socket
         hostPath:
-          path: {{ $comp.fullWorkspaceBackup.containerd.socket }}
+          path: {{ $comp.containerRuntime.containerd.socket }}
           type: Socket
-      {{- range $idx, $pth := $comp.fullWorkspaceBackup.nodeRoots }}
+      {{- range $idx, $pth := $comp.containerRuntime.nodeRoots }}
       - name: node-fs{{ $idx }}
         hostPath:
           path: {{ $pth }}
@@ -79,7 +79,14 @@ spec:
         hostPath:
           path: /proc/mounts
           type: File
-      {{- end }}
+      - name: node-cgroups
+        hostPath:
+          path: {{ $comp.cgroupsRoot | default "/sys/fs/cgroup" }}
+          type: Directory
+      - name: node-hosts
+        hostPath:
+          path: {{ $comp.nodeHosts | default "/etc/hosts" }}
+          type: File
       {{- if $comp.userNamespaces.shiftfsModuleLoader.enabled }}
       - name: node-linux-src
         hostPath: 
@@ -108,18 +115,21 @@ spec:
           mountPropagation: Bidirectional
         - mountPath: /config
           name: config
-        {{- if $comp.fullWorkspaceBackup.enabled }}
         - mountPath: /mnt/containerd.sock
           name: containerd-socket
-        {{- range $idx, $pth := $comp.fullWorkspaceBackup.nodeRoots }}
+        {{- range $idx, $pth := $comp.containerRuntime.nodeRoots }}
         - mountPath: /mnt/node{{ $idx }}
           name: node-fs{{ $idx }}
         {{- end }}
         - mountPath: /mnt/mounts
           name: node-mounts
           readOnly: true
           mountPropagation: HostToContainer
-        {{- end }}
+        - mountPath: /mnt/node-cgroups
+          name: node-cgroups
+          mountPropagation: HostToContainer
+        - mountPath: /mnt/hosts
+          name: node-hosts
         - mountPath: /certs
           name: tls-certs
 {{- if $comp.volumeMounts }}
@@ -139,6 +149,9 @@ spec:
         - name: rpc
           containerPort: {{ $comp.servicePort }}
           hostPort: {{ $comp.servicePort }}
+        - name: registry-proxy
+          containerPort: {{ $comp.registryProxyPort }}
+          hostPort: {{ $comp.registryProxyPort }}
         - name: metrics
           containerPort: 9500
         securityContext:

diff --git a/chart/templates/ws-daemon-networkpolicy.yaml b/chart/templates/ws-daemon-networkpolicy.yaml
@@ -1,7 +1,7 @@
 # Copyright (c) 2020 TypeFox GmbH. All rights reserved.
 # Licensed under the MIT License. See License-MIT.txt in the project root for license information.
 
-{{ $comp := .Values.components.wsSync -}}
+{{ $comp := .Values.components.wsDaemon -}}
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -23,7 +23,6 @@ spec:
     - protocol: TCP
       port: {{ $comp.servicePort }}
     from:
-    # Allow ingress on port 3000 from component:
     - podSelector:
         matchLabels:
           app: {{ template "gitpod.fullname" . }}

diff --git a/chart/templates/ws-daemon-tlssecret.yaml b/chart/templates/ws-daemon-tlssecret.yaml
@@ -1,10 +1,10 @@
 # Copyright (c) 2020 TypeFox GmbH. All rights reserved.
 # Licensed under the MIT License. See License-MIT.txt in the project root for license information.
 
-{{ $comp := .Values.components.wsSync -}}
+{{ $comp := .Values.components.wsDaemon -}}
 {{- $this := dict "root" . "gp" $.Values "comp" $comp -}}
 {{- if not $comp.disabled -}}
-{{- $altNames := list ( printf "%s.%s" (include "gitpod.fullname" .) .Release.Namespace ) ( printf "%s.%s.svc" "ws-daemon" .Release.Namespace ) ( printf "wssync" ) -}}
+{{- $altNames := list ( printf "%s.%s" (include "gitpod.fullname" .) .Release.Namespace ) ( printf "%s.%s.svc" "ws-daemon" .Release.Namespace ) ( printf "wsdaemon" ) -}}
 {{- $ca := genCA "wssync-ca" 365 -}}
 {{- $cert := genSignedCert (include "gitpod.fullname" . ) nil $altNames 365 $ca -}}
 apiVersion: v1

diff --git a/chart/templates/ws-manager-configmap.yaml b/chart/templates/ws-manager-configmap.yaml
@@ -22,8 +22,8 @@ data:
             "namespace": "{{ .Release.Namespace }}",
             "schedulerName": "{{ .Values.workspaceScheduler }}",
             "dryRun": false,
-            "wssync": {
-                "port": {{ .Values.components.wsSync.servicePort }},
+            "wsdaemon": {
+                "port": {{ .Values.components.wsDaemon.servicePort }},
                 "tls": {
                     "ca": "/certs/ca.crt",
                     "crt": "/certs/tls.crt",
@@ -70,7 +70,7 @@ data:
             "urlTemplate":     "{{- template "gitpod.scheme" $this -}}://{{ $.Values.hostname }}/workspace/{{"{{ .Prefix }}"}}/",
             "portUrlTemplate": "{{- template "gitpod.scheme" $this -}}://{{"{{ .WorkspacePort }}"}}-{{"{{ .Prefix }}"}}.ws{{- if $.Values.installation.shortname -}}-{{ $.Values.installation.shortname }}{{- end -}}.{{ $.Values.hostname }}",
             {{- end }}
-            "workspaceHostPath": "{{ .Values.components.wsSync.hostWorkspaceArea }}",
+            "workspaceHostPath": "{{ .Values.components.wsDaemon.hostWorkspaceArea }}",
             "theiaHostPath": "{{ .Values.components.nodeDaemon.theiaHostBasePath }}/theia/theia-{{ .Values.version }}",
             "podTemplate": {
                 {{- if $wscomp.templates }}
@@ -92,7 +92,7 @@ data:
             },
             {{ if $comp.eventTraceLogLocation }}"eventTraceLog": "{{ $comp.eventTraceLogLocation }}",{{- end }}
             "reconnectionInterval": "30s",
-            "registryFacadeHost": {{ (printf "reg.%s:%v" (.Values.components.registryFacade.hostname | default .Values.hostname) .Values.components.wsManagerNode.registryProxyPort) | quote }}
+            "registryFacadeHost": {{ (printf "reg.%s:%v" (.Values.components.registryFacade.hostname | default .Values.hostname) .Values.components.wsDaemon.registryProxyPort) | quote }}
             {{- if and (not $wsproxy.disabled) (eq .Values.ingressMode "noDomain") -}}
             , "ingressPortAllocator": {
                 "ingressRange": {
@@ -105,7 +105,7 @@ data:
             {{ if $comp.additionalConfig }}, {{ $comp.additionalConfig | toJson | trim | trimPrefix "{" | trimSuffix "}" }}{{- end }}
         },
         "content": {
-            "storage": {{ toJson .Values.components.wsSync.remoteStorage }}
+            "storage": {{ toJson .Values.components.wsDaemon.remoteStorage }}
         },
         "rpcServerAddr": ":8080",
         "pprofAddr": "localhost:6060",