more flexibility to define connectivity options

interTwin-eu · Nov 13, 2024 · ec67bb3 · ec67bb3
1 parent a6ea49c
commit ec67bb3
Show file tree

Hide file tree

Showing 3 changed files with 98 additions and 39 deletions.
diff --git a/interlink/templates/virtual-kubelet-config.yaml b/interlink/templates/virtual-kubelet-config.yaml
@@ -5,14 +5,19 @@ metadata:
   namespace: {{ .Release.Namespace }}
 data:
   InterLinkConfig.yaml: |
-    InterlinkURL: "{{ if .Values.interlink.socket }}{{ .Values.interlink.socket }}{{ else }}{{ .Values.interlink.address }}{{ end }}"
-    InterlinkPort: {{ if .Values.interlink.socket }}-1{{ else }}{{ .Values.interlink.port }}{{ end }}
-    ExportPodData: true
+    {{- if .Values.interlink.socket }}
+    InterlinkURL: {{ .Values.interlink.socket | quote }}
+    InterlinkPort: {{ printf "%s-1" .Values.interlink.socket | quote }}
+    {{- else }}
+    InterlinkURL: {{ .Values.interlink.address | quote }}
+    InterlinkPort: {{ .Values.interlink.port | quote }}
+    {{- end }}
+    ExportPodData: {{ .Values.interlink.exportPodData }}
     VerboseLogging: true
     ErrorsOnlyLogging: false
     ServiceAccount: "{{ .Values.nodeName }}"
     Namespace: "{{ .Release.Namespace }}"
-    VKTokenFile: {{ if .Values.interlink.socket }}/dev/null{{ else }}/opt/interlink/token{{ end }}
+    VKTokenFile: {{ .Values.OAUTH.enabled | ternary "/opt/interlink/token" "/dev/null" }}
     CPU: "{{ .Values.virtualNode.CPUs }}"
     Memory: "{{ .Values.virtualNode.MemGiB }}Gi"
     Pods: "{{ .Values.virtualNode.Pods }}"
@@ -21,21 +26,30 @@ data:
     KubeletHTTP:
       Insecure: {{ .Values.virtualNode.KubeletHTTP.Insecure }}
 ---
-{{- if .Values.interlink.socket }}
+{{- if .Values.interlink.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: "{{ .Values.nodeName }}-interlink-config"
   namespace: {{ .Release.Namespace }}
 data:
   InterLinkConfig.yaml: |
-    InterlinkAddress: "{{ .Values.interlink.socket }}"
-    SidecarURL: "{{ if .Values.plugin.socket }}{{ .Values.plugin.socket }}{{ else }}{{ .Values.plugin.address }}{{ end }}"
-    SidecarPort: "{{ .Values.plugin.port }}"
+    {{- if .Values.interlink.socket }}
+    InterlinkAddress: {{ .Values.interlink.socket | quote }}
+    {{- else }}
+    InterlinkAddress: {{ .Values.interlink.address | quote }}
+    InterlinkPort: {{ .Values.interlink.port | quote }}
+    {{- end }}
+    {{- if .Values.plugin.socket }}
+    SidecarURL: {{ .Values.plugin.socket | quote }}
+    {{- else }}
+    SidecarURL: {{ .Values.plugin.address | quote }}
+    SidecarPort: {{ .Values.plugin.port | quote }}
+    {{- end }}
     VerboseLogging: true
     ErrorsOnlyLogging: false
-    ExportPodData: true
-    DataRootFolder: "~/.interlink"
+    ExportPodData: {{ .Values.interlink.exportPodData }}
+    DataRootFolder: /data/interlink
 {{- end }}
 ---
 {{- if .Values.plugin.enabled }}

diff --git a/interlink/templates/virtual-kubelet.yaml b/interlink/templates/virtual-kubelet.yaml
@@ -13,27 +13,37 @@ spec:
   template:
     metadata:
       labels:
-        app: {{ .Values.nodeName }} 
+        app: {{ .Values.nodeName }}
     spec:
       automountServiceAccountToken: true
-      serviceAccountName: {{ .Values.nodeName }} 
+      serviceAccountName: {{ .Values.nodeName }}
       containers:
       {{- if .Values.plugin.enabled }}
       - name: plugin 
         image: "{{ .Values.plugin.image }}"
         imagePullPolicy: Always
-        commands: {{ .Values.plugin.commands }} 
-        args: {{ .Values.plugin.args }} 
+        {{- with .Values.plugin.commands }}
+        commands:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.plugin.args }}
+        args:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.plugin.envs }}
         env:
-          - name: INTERLINKCONFIGPATH
-            value: "/etc/interlink/plugin.yaml"
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
         volumeMounts:
-          - name: il-config mountPath: /etc/interlink/plugin.yaml
+          - name: plugin-config
+            mountPath: /etc/interlink/plugin.yaml
             subPath: plugin.yaml
+          {{- if .Values.plugin.socket }}
           - name: sockets
             mountPath: /var/run/
+          {{- end }}
       {{- end }}
-      {{- if .Values.interlink.socket }}
+      {{- if .Values.interlink.enabled }}
       - name: interlink
         image: "{{ .Values.interlink.image }}"
         imagePullPolicy: Always
@@ -44,15 +54,21 @@ spec:
           - name: il-config
             mountPath: /etc/interlink/InterLinkConfig.yaml
             subPath: InterLinkConfig.yaml
+          {{- if or .Values.interlink.socket .Values.plugin.socket }}
           - name: sockets
             mountPath: /var/run/
+          {{- end }}
+          {{- if .Values.interlink.exportPodData }}
+          - name: {{ .Values.interlink.dataRootVolume }}
+            mountPath: /data/interlink
+          {{- end }}
       {{- end }}
       - name: vk
-        image: {{ .Values.virtualNode.image }} 
+        image: {{ .Values.virtualNode.image }}
         imagePullPolicy: Always
         env:
         - name: NODENAME
-          value: {{ .Values.nodeName }} 
+          value: {{ .Values.nodeName }}
         - name: KUBELET_PORT
           value: "10250"
         - name: POD_IP
@@ -61,8 +77,10 @@ spec:
               fieldPath: status.podIP
         - name: CONFIGPATH
           value: "/etc/interlink/InterLinkConfig.yaml"
+        {{- if .Values.OAUTH.enabled }}
         - name: VKTOKENFILE
           value: "/opt/interlink/token"
+        {{- end }}
         {{- if .Values.virtualNode.HTTPProxies.HTTP }}
         - name: HTTP_PROXY
           value: {{ .Values.virtualNode.HTTPProxies.HTTP }}
@@ -75,12 +93,15 @@ spec:
         - name: config
           mountPath: /etc/interlink/InterLinkConfig.yaml
           subPath: InterLinkConfig.yaml
+        {{- if .Values.OAUTH.enabled }}
         - name: token
           mountPath: /opt/interlink
-      {{- if .Values.interlink.socket }}
+        {{- end }}
+        {{- if .Values.interlink.socket }}
         - name: sockets
           mountPath: /var/run/
-      {{- else }}
+        {{- end }}
+      {{- if .Values.OAUTH.enabled }}
       - name: refresh-token
         image: "{{ .Values.OAUTH.image }}"
         imagePullPolicy: Always
@@ -145,6 +166,12 @@ spec:
         #   subPath: sshd_config
       {{- end }}
       volumes:
+      - name: config
+        configMap:
+          name: {{ .Values.nodeName }}-virtual-kubelet-config
+      {{- with .Values.extraVolumes }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
       {{- if .Values.sshBastion.enabled }}
       - name: ssh-keys
         secret:
@@ -153,20 +180,28 @@ spec:
         configMap:
           name: "{{ .Values.nodeName }}-sshd-config"
       {{- end }}
-      - name: config
-        configMap:
-          name: {{ .Values.nodeName }}-virtual-kubelet-config
+      {{- if .Values.OAUTH.enabled }}
       - name: token
         hostPath:
           path: /tmp
           type: Directory
-      {{- if .Values.interlink.socket }}
+      {{- end }}
+      {{- if .Values.interlink.enabled }}
       - name: il-config
         configMap:
           name: {{ .Values.nodeName }}-interlink-config
+      {{- end }}
+      {{- if .Values.plugin.enabled }}
+      - name: plugin-config
+        configMap:
+          name: {{ .Values.nodeName }}-plugin-config
+      {{- end }}
+      {{- if or .Values.interlink.socket .Values.plugin.socket }}
       - name: sockets
-        emptyDir: {}  
+        emptyDir: {}
       {{- end }}
+
+{{- if .Values.sshBastion.enabled }}
 ---
 apiVersion: v1
 kind: Secret
@@ -176,9 +211,8 @@ metadata:
 type: Opaque
 data:
   authorized_keys: {{ .Values.sshBastion.clientKeys.authorizedKeys | b64enc }}
----
 
-{{- if .Values.sshBastion.enabled }}
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -193,7 +227,6 @@ spec:
     app: {{ .Values.nodeName }}
 
 ---
-
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -206,5 +239,3 @@ data:
     AllowTcpForwarding yes
 
 {{- end }}
-
----
diff --git a/interlink/values.yaml b/interlink/values.yaml
@@ -5,10 +5,13 @@
 nodeName: virtual-node
 
 interlink:
-  image: ghcr.io/intertwin-eu/interlink/interlink:latest 
+  image: ghcr.io/intertwin-eu/interlink/interlink:latest
   enabled: false
-  socket: null 
-  # if socket is specified, the following fields are ignored
+  exportPodData: false
+  dataRootVolume: ""
+  # unix socket in /var/run
+  socket: null
+  # if socket is specified, the address/port are ignored
   address: http://localhost
   port: 4000
 
@@ -30,11 +33,15 @@ plugin:
   image: "" 
   commands: [] 
   args: []
-  envs: []
+  # content of the "config" will be mounted as /etc/interlink/plugin.yaml
   config: ""
-  address: ""  
-  port: 3000 
+  envs: []
+    # - name: SLURMCONFIGPATH
+    #   value: /etc/interlink/plugin.yaml
   socket: null
+  # if socket is specified, the address/port are ignored
+  address: ""
+  port: 3000
 
 sshBastion:
   enabled: false
@@ -44,8 +51,9 @@ sshBastion:
     keysURL: ""
   port: 31022
 
-# if interlink.socket is specified, the following fields are ignored
+# disable OAUTH when using sockets for communication in the in-cluster mode
 OAUTH:
+  enabled: false
   image: ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw-refresh:latest
   TokenURL: DUMMY
   ClientID: DUMMY
@@ -54,3 +62,9 @@ OAUTH:
   GrantType: authorization_code
   Audience: DUMMY
 
+# define extra volumes (e.g. for DataRootVolume)
+extraVolumes: []
+  # - name: data-root
+  #   hostPath:
+  #     path: /scratch/interlink/
+  #     type: DirectoryOrCreate