From fedb16f5468b0b02792dad536179c8c6d0695997 Mon Sep 17 00:00:00 2001 From: yatarkan Date: Tue, 28 Jan 2025 19:26:16 +0400 Subject: [PATCH] Propagate stop token ids from default generation config in CB --- src/cpp/src/continuous_batching_impl.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cpp/src/continuous_batching_impl.cpp b/src/cpp/src/continuous_batching_impl.cpp index 99df043090..3e2fc9c98f 100644 --- a/src/cpp/src/continuous_batching_impl.cpp +++ b/src/cpp/src/continuous_batching_impl.cpp @@ -149,6 +149,9 @@ GenerationHandle ContinuousBatchingPipeline::ContinuousBatchingImpl::add_request(uint64_t request_id, const ov::Tensor& input_ids, ov::genai::GenerationConfig sampling_params) { + // If stop_token_ids were not provided, take value from default m_generation_config + if (sampling_params.stop_token_ids.empty()) + sampling_params.stop_token_ids = m_generation_config.stop_token_ids; // If eos_token_id was not provided, take value from default m_generation_config if (sampling_params.eos_token_id == -1) sampling_params.set_eos_token_id(m_generation_config.eos_token_id);