fix: include add_special_tokens in kserve request (huggingface#2859)

merging as this patch is already used, and fully limit to the kserve feature
khulaifi95 · Dec 19, 2024 · 23bc38b · 23bc38b
1 parent ab5f616
commit 23bc38b
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/router/src/kserve.rs b/router/src/kserve.rs
@@ -205,14 +205,15 @@ pub async fn kserve_model_infer(
             let generate_request = GenerateRequest {
                 inputs: str_input.to_string(),
                 parameters: payload.parameters.clone(),
+                add_special_tokens: true,
             };
             let infer = infer.clone();
             let compute_type = compute_type.clone();
             let span = tracing::Span::current();
             async move {
                 generate_internal(infer, compute_type, Json(generate_request), span)
                     .await
-                    .map(|(_, Json(generation))| {
+                    .map(|(_, _, Json(generation))| {
                         let generation_as_bytes = generation.generated_text.as_bytes().to_vec();
                         OutputChunk {
                             name: output.name.clone(),