Fix auto-scaling v2

kubernetes/charts/oasis-models/templates/workers.yaml

@@ -1,6 +1,6 @@
 {{- $root := . -}}
 {{- range $k, $worker := .Values.workers }}
-{{- $name := printf "worker-%s-%s-%s" $worker.supplierId $worker.modelId $worker.modelVersionId | lower }}
+{{- $name := printf "worker-%s-%s-%s-v2" $worker.supplierId $worker.modelId $worker.modelVersionId | lower }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:

kubernetes/charts/oasis-models/templates/workers_v1.yaml

@@ -0,0 +1,107 @@
+{{- $root := . -}}
+{{- range $k, $worker := .Values.workers }}
+{{- $name := printf "worker-%s-%s-%s" $worker.supplierId $worker.modelId $worker.modelVersionId | lower }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $name | quote }}
+  labels:
+    {{- include "h.labels" $root | nindent 4 }}
+    {{- include "h.workerLabels" . | nindent 4 }}
+  annotations:
+    {{- include "h.workerDeploymentAnnotations" . | indent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ $name }}
+  strategy:
+    type: Recreate
+  replicas: 0
+  template:
+    metadata:
+      labels:
+        app: {{ $name }}
+        {{- include "h.labels" $root | nindent 8 }}
+        {{- include "h.workerLabels" $worker | nindent 8 }}
+      annotations:
+        checksum/oasis-server: {{ toJson $root.Values.oasisServer | sha256sum }}
+        checksum/model_registration: {{ $root.Files.Get "resources/model_registration.sh" | sha256sum }}
+    spec:
+      {{- include "h.affinity" $root | nindent 6 }}
+      initContainers:
+        {{- include "h.initTcpAvailabilityCheckBySecret" (list $root $root.Values.databases.broker.name $root.Values.databases.celery_db.name) | nindent 8}}
+      containers:
+        - image: {{ .image }}:{{ .version }}
+          {{- if .imagePullPolicy }}
+          imagePullPolicy: {{ .imagePullPolicy }}
+          {{- end }}
+          name: worker
+          env:
+            {{- include "h.modelEnvs" . | indent 12 }}
+            {{- include "h.brokerVars" $root | indent 12 }}
+            {{- include "h.celeryDbVars" $root | indent 12}}
+          startupProbe:
+            exec:
+              command: ["celery", "-A", "src.model_execution_worker.singleserver_tasks", "inspect", "ping"]
+            timeoutSeconds: 5
+            periodSeconds: 10
+            failureThreshold: 30
+          livenessProbe:
+            exec:
+              command: ["celery", "-A", "src.model_execution_worker.singleserver_tasks", "inspect", "ping"]
+            initialDelaySeconds: 30
+            periodSeconds: 60
+            timeoutSeconds: 60
+            failureThreshold: 15
+          volumeMounts:
+            - name: shared-fs-persistent-storage
+              mountPath: /shared-fs
+            {{- range .volumes }}
+            - name: {{ .name }}
+              {{- if eq .source "azureFiles" }}
+              subPath: {{ printf "%s/%s/%s" $worker.supplierId $worker.modelId $worker.modelVersionId }}
+              {{- end }}
+              mountPath: {{ .mountPath }}
+            {{- end }}
+{{- if .registryCredentials }}
+      imagePullSecrets:
+        - name: {{ .registryCredentials }}
+{{- end }}
+{{- if ($root.Values.azure).secretProvider }}
+            - name: azure-secret-provider
+              mountPath: "/mnt/azure-secrets-store"
+              readOnly: true
+{{- end }}
+      volumes:
+        - name: shared-fs-persistent-storage
+{{- if (($root.Values.volumes.host).sharedFs) }}
+          persistentVolumeClaim:
+            claimName: {{ $root.Values.volumes.host.sharedFs.name }}
+{{- else if (($root.Values.volumes.azureFiles).sharedFs) }}
+{{- toYaml $root.Values.volumes.azureFiles.sharedFs | nindent 10 }}
+{{- end }}
+        {{- range .volumes }}
+        - name: {{ .name }}
+          {{- if eq .source "host" }}
+          persistentVolumeClaim:
+            claimName: {{ .name }}
+          {{- else if eq .source "azureFiles" }}
+            {{- $workerVolumeName := .name }}
+            {{- range $root.Values.modelVolumes }}
+              {{- if eq .name $workerVolumeName }}
+          csi:
+            {{- toYaml .csi | nindent 12 }}
+              {{- end }}
+            {{- end }}
+          {{- end }}
+        {{- end }}
+{{- if ($root.Values.azure).secretProvider }}
+        - name: azure-secret-provider
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: "azure-secret-provider"
+{{- end }}
+---
+{{- end }}

kubernetes/charts/oasis-platform/templates/oasis_worker_controller.yaml

@@ -78,6 +78,8 @@ spec:
               value: {{ .Values.oasisServer.name }}
             - name: OASIS_API_PORT
               value: {{ .Values.oasisServer.port | quote }}
+            - name: OASIS_API_SUBPATH
+              value: "api"
             - name: OASIS_WEBSOCKET_HOST
               value: {{ .Values.oasisWebsocket.name }}
             - name: OASIS_WEBSOCKET_PORT

kubernetes/worker-controller/src/oasis_client.py

@@ -15,7 +15,7 @@ class OasisClient:
     A simple client for the Oasis API. Takes care of the access token and supports searching for models.
     """
 
-    def __init__(self, http_host, http_port, ws_host, ws_port, secure, username, password):
+    def __init__(self, http_host, http_port, http_subpath, ws_host, ws_port, secure, username, password):
         """
         :param http_host: Oasis API hostname.
         :param http_port: Oasis API port.
@@ -25,17 +25,21 @@ def __init__(self, http_host, http_port, ws_host, ws_port, secure, username, pas
         :param username: Username for API authentication.
         :param password: Password for API authentication.
         """
-        self.host = http_host
-        self.port = http_port
         self.ws_host = ws_host
         self.ws_port = ws_port
-
         self.secure = secure
-        self.http_host = ('https://' if secure else 'http://') + f'{http_host}' + (f':{http_port}' if http_port else '')
+
+        api_proto = 'https://' if secure else 'http://'
+        api_host = http_host
+        api_port = f':{http_port}' if http_port else ''
+        api_path = f'/{http_subpath}' if http_subpath else ''
+        self.http_host = f'{api_proto}{api_host}{api_port}{api_path}'
+
         self.username = username
         self.password = password
         self.access_token = None
         self.token_expire_time = None
+        print('Connecting to: ' + self.http_host)
 
     def is_authenticated(self) -> bool:
         """

kubernetes/worker-controller/src/worker_controller.py

@@ -56,6 +56,7 @@ def parse_args():
     parser = argparse.ArgumentParser('Oasis example model worker controller')
     parser.add_argument('--api-host', help='The sever API hostname', default=getenv('OASIS_API_HOST', default='localhost'))
     parser.add_argument('--api-port', help='The server API portnumber', default=getenv('OASIS_API_PORT', default=8000))
+    parser.add_argument('--api-subpath', help='The server API subpath, e.g. "api"', default=getenv('OASIS_API_SUBPATH', default=''))
     parser.add_argument('--websocket-host', help='The websocket hostname', default=getenv('OASIS_WEBSOCKET_HOST', default='localhost'))
     parser.add_argument('--websocket-port', help='The websocket portnumber', default=getenv('OASIS_WEBSOCKET_PORT', default=8001))
     parser.add_argument('--secure', help='Flag if https and wss should be used', default=bool(getenv('OASIS_API_SECURE')), action='store_true')
@@ -96,6 +97,7 @@ def main():
     oasis_client = OasisClient(
         cli_args.api_host,
         cli_args.api_port,
+        cli_args.api_subpath,
         cli_args.websocket_host,
         cli_args.websocket_port,
         cli_args.secure,