use templates and config maps to get the settings for sidekiq the way…

… we want
notch8 · Feb 24, 2024 · 95d3d3d · 95d3d3d
1 parent 2dacedf
commit 95d3d3d
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 31 deletions.
diff --git a/bin/worker b/bin/worker
@@ -4,13 +4,9 @@ if ENV['GOOGLE_OAUTH_PRIVATE_KEY_VALUE'] && !ENV['GOOGLE_OAUTH_PRIVATE_KEY_VALUE
 end
 
 if ENV['DATABASE_URL'] && !ENV['DATABASE_URL'].empty?
-  ENV['DATABASE_URL'] = ENV['DATABASE_URL'].gsub('pool=5', 'pool=30')
+  ENV['DATABASE_URL'] = ENV['DATABASE_URL'].gsub('pool=5', "pools=#{ENV.fetch('WORKER_CONCURENCY'), 30}")
 else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-if ENV['SIDEKIQ_CONFIG']
-  exec "echo $DATABASE_URL && bundle exec sidekiq -C #{ENV['SIDEKIQ_CONFIG']}"
-else
-  exec "echo $DATABASE_URL && bundle exec sidekiq"
-end
+exec "echo $DATABASE_URL && RAILS_MAX_THREADS=$WORKER_CONCURENCY bundle exec sidekiq $WORKER_ARGS"
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
diff --git a/config/sidekiq_auxiliary.yml b/config/sidekiq_auxiliary.yml
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
@@ -187,6 +187,9 @@ worker:
       cpu: "150m"
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars
+  extraEnvFrom:
+    - configMapRef:
+        name: worker-env
   podSecurityContext:
     runAsUser: 1001
     runAsGroup: 101
@@ -204,14 +207,35 @@ workerAuxiliary:
     requests:
       memory: "4Gi"
       cpu: "2"
-  extraEnvVars:
-    - name: AUXILIARY_WORKER_THREAD_COUNT
-      value: "1"
-    - name: SIDEKIQ_CONFIG
-      value: "config/sidekiq_auxiliary.yml"
+  extraEnvVars: *envVars
+  extraEnvFrom:
+    - configMapRef:
+        name: auxilary-worker-env
+
 
 extraDeploy:
   - |-
+      ---
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: worker-env
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      data:
+        WORKER_CONCURENCY: "5"
+        WORKER_ARGS: -q default -q import -q export
+      ---
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: auxiliary-worker-env
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      data:
+        WORKER_CONCURENCY: "1"
+        WORKER_ARGS: -q auxiliary -q default -q import -q export
+      ---
       apiVersion: apps/v1
       kind: Deployment
       metadata:
@@ -241,9 +265,11 @@ extraDeploy:
                       name: {{ include "hyrax.fullname" . }}-env
                   - secretRef:
                       name: {{ template "hyrax.fullname" . }}
+                  {{- with .Values.workerAuxiliary.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
                 env:
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
-                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
                 command:
                   - sh
                   - -c
@@ -273,12 +299,11 @@ extraDeploy:
                   - secretRef:
                       name: {{ .Values.solrExistingSecret }}
                   {{- end }}
-                  {{- with .Values.worker.extraEnvFrom }}
+                  {{- with .Values.workerAuxiliary.extraEnvFrom }}
                     {{- toYaml . | nindent 12 }}
                   {{- end }}
                 env:
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
-                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
                 {{- if .Values.worker.readinessProbe.enabled }}
                 readinessProbe:
                   exec:

diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
@@ -203,6 +203,9 @@ worker:
       cpu: "150m"
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars
+  extraEnvFrom:
+    - configMapRef:
+        name: worker-env
   podSecurityContext:
     runAsUser: 1001
     runAsGroup: 101
@@ -220,14 +223,34 @@ workerAuxiliary:
     requests:
       memory: "4Gi"
       cpu: "2"
-  extraEnvVars:
-    - name: AUXILIARY_WORKER_THREAD_COUNT
-      value: "1"
-    - name: SIDEKIQ_CONFIG
-      value: "config/sidekiq_auxiliary.yml"
+  extraEnvVars: *envVars
+  extraEnvFrom:
+    - configMapRef:
+        name: auxilary-worker-env
 
 extraDeploy:
   - |-
+      ---
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: worker-env
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      data:
+        WORKER_CONCURENCY: "5"
+        WORKER_ARGS: -q default -q import -q export
+      ---
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: auxiliary-worker-env
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      data:
+        WORKER_CONCURENCY: "1"
+        WORKER_ARGS: -q auxiliary -q default -q import -q export
+      ---
       apiVersion: apps/v1
       kind: Deployment
       metadata: