✨🔧 configurable model by cli in server mode

chriamue · Jan 8, 2024 · 846ddd0 · 846ddd0
1 parent e5b4380
commit 846ddd0
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -99,6 +99,7 @@ The following table shows the performance metrics of the model on different syst
 | 7b-open-chat-3.5 | AMD 7900X3D (12 Core) 64GB | 9.4 tokens/s      |
 | 7b-open-chat-3.5 | AMD 5600G (8 Core VM) 16GB | 2.8 tokens/s      |
 | 13b (llama2 13b) | AMD 7900X3D (12 Core) 64GB | 5.2 tokens/s      |
+| phi-2            | AMD 7900X3D (12 Core) 64GB | 20.6 tokens/s     |
 
 ## Todo
 
@@ -108,3 +109,4 @@ The following table shows the performance metrics of the model on different syst
 - [x] docker image and docker-compose
 - [ ] add tests
 - [ ] add documentation
+- [ ] fix stop token
diff --git a/src/main.rs b/src/main.rs
@@ -90,7 +90,9 @@ async fn main() {
     let opt = Opt::parse();
 
     match load_config(&opt.config) {
-        Ok(config) => {
+        Ok(mut config) => {
+            config.model = opt.model.unwrap_or(config.model);
+
             info!("Loaded config: {:?}", config);
             if let Some(prompt) = opt.prompt {
                 let parameter = GenerateParameter {