Allow eager_prefill to be set in Helm chart (#557)

predibase · Jul 26, 2024 · 15a38d5 · 15a38d5
1 parent 2e81331
commit 15a38d5
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/charts/lorax/templates/deployment.yaml b/charts/lorax/templates/deployment.yaml
@@ -45,6 +45,8 @@ spec:
         - {{ .Values.deployment.args.maxBatchPrefillTokens | quote }}
         - --sharded
         - {{ .Values.deployment.args.sharded | quote }}
+        - --eager-prefill
+        - {{ .Values.deployment.args.eagerPrefill | quote }}
         env:
         - name: PORT
           value: "8000"

diff --git a/charts/lorax/values.yaml b/charts/lorax/values.yaml
@@ -13,6 +13,7 @@ deployment:
     maxBatchTotalTokens: 4096
     maxBatchPrefillTokens: 2048
     sharded: false
+    eagerPrefill: false
 
   env:
     # Your huggingface hub token. Required for some models such as the llama-2 family.