🔥 removes token output stream to use it from candle examples

Cargo.toml

@@ -17,6 +17,7 @@ path = "src/main.rs"
 anyhow = "1.0"
 axum = "0.7"
 candle-core = "0.3.2"
+candle-examples = "0.3.2"
 candle-transformers = "0.3.2"
 clap = { version = "4.4", features = ["derive"] }
 futures = "0.3.29"

src/llm/mod.rs

@@ -6,7 +6,6 @@ pub mod sampler;
 mod text_generation;
 pub mod text_generator;
 pub mod token_generator;
-pub mod token_output_stream;
 
 pub use text_generator::TextGeneratorTrait;
 

src/llm/text_generation.rs

@@ -6,6 +6,7 @@ use crate::{
 use crate::llm::generate_parameter::GenerateParameter;
 use anyhow::Result;
 use candle_core::Device;
+use candle_examples::token_output_stream::TokenOutputStream;
 use candle_transformers::{generation::LogitsProcessor, models::quantized_llama::ModelWeights};
 use futures::Stream;
 use log::{info, trace};
@@ -17,7 +18,6 @@ use tokio_stream::wrappers::ReceiverStream;
 use super::{
     text_generator::{self, TextGenerator},
     token_generator::{TokenGenerator, TokenGeneratorTrait},
-    token_output_stream::TokenOutputStream,
     TextGeneratorTrait,
 };
 

src/llm/text_generator/mod.rs

@@ -1,9 +1,9 @@
 use super::{
     token_generator::{TokenGeneratorResult, TokenGeneratorTrait},
-    token_output_stream::TokenOutputStream,
     FinishReason,
 };
 use anyhow::Result;
+use candle_examples::token_output_stream::TokenOutputStream;
 mod dummy_text_generator;
 
 pub type TextProbability = (String, f32);