Skip to content

Commit

Permalink
add in response headers to streaming endpoint (#282)
Browse files Browse the repository at this point in the history
  • Loading branch information
noyoshi authored Feb 26, 2024
1 parent 54007ac commit 362ee41
Showing 1 changed file with 22 additions and 0 deletions.
22 changes: 22 additions & 0 deletions router/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,28 @@ async fn generate_stream_with_callback(
});
}

let (adapter_source, adapter_parameters) = extract_adapter_params(
req.0.parameters.adapter_id.clone(),
req.0.parameters.adapter_source.clone(),
req.0.parameters.adapter_parameters.clone(),
);

let adapter_id_string = adapter_parameters
.adapter_ids
.iter()
.map(|id| id.as_str())
// filter out base model adapter id
.filter(|id| *id != BASE_MODEL_ADAPTER_ID)
.collect::<Vec<_>>()
.join(",");

if adapter_id_string.len() > 0 {
headers.insert("x-adapter-ids", adapter_id_string.parse().unwrap());
headers.insert("x-adapter-source", adapter_source.unwrap().parse().unwrap());
}

headers.insert("x-model-id", MODEL_ID.get().unwrap().parse().unwrap());

let stream = async_stream::stream! {
// Inference
let mut end_reached = false;
Expand Down

0 comments on commit 362ee41

Please sign in to comment.