1. ignore some languages; 2. improve log for debug.

yiwen-ai · Aug 21, 2023 · d23d55a · d23d55a
1 parent 3d19b22
commit d23d55a
Show file tree

Hide file tree

Showing 7 changed files with 31 additions and 22 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "jarvis"
-version = "0.7.10"
+version = "0.7.11"
 edition = "2021"
 rust-version = "1.64"
 description = ""

diff --git a/src/api/embedding.rs b/src/api/embedding.rs
@@ -455,7 +455,7 @@ pub async fn public(
                     rid = rid,
                     gid = gid.to_string(),
                     cid = cid.to_string(),
-                    language = language.to_string(),
+                    language = language.to_639_3().to_string(),
                     elapsed = start.elapsed().as_millis() as u64;
                     "success",
                 )
@@ -466,7 +466,7 @@ pub async fn public(
                     rid = rid,
                     gid = gid.to_string(),
                     cid = cid.to_string(),
-                    language = language.to_string(),
+                    language = language.to_639_3().to_string(),
                     elapsed = start.elapsed().as_millis() as u64;
                     "{}", err,
                 )

diff --git a/src/api/summarizing.rs b/src/api/summarizing.rs
@@ -172,7 +172,7 @@ async fn summarize(
                     rid = &rid,
                     gid = gid.to_string(),
                     cid = cid.to_string(),
-                    language = language.to_string(),
+                    language = language.to_639_3().to_string(),
                     version = version,
                     elapsed = ai_elapsed;
                     "{}", err.to_string(),
@@ -185,7 +185,7 @@ async fn summarize(
                     rid = &rid,
                     gid = gid.to_string(),
                     cid = cid.to_string(),
-                    language = language.to_string(),
+                    language = language.to_639_3().to_string(),
                     version = version,
                     elapsed = ai_elapsed;
                     "success",
@@ -212,7 +212,7 @@ async fn summarize(
                 rid = &rid,
                 gid = gid.to_string(),
                 cid = cid.to_string(),
-                language = language.to_string(),
+                language = language.to_639_3().to_string(),
                 version = version,
                 elapsed = start.elapsed().as_millis() as u64,
                 summary = doc.summary.len(),
@@ -226,7 +226,7 @@ async fn summarize(
                 rid = &rid,
                 gid = gid.to_string(),
                 cid = cid.to_string(),
-                language = language.to_string(),
+                language = language.to_639_3().to_string(),
                 version = version,
                 elapsed = start.elapsed().as_millis() as u64,
                 summary = doc.summary.len(),

diff --git a/src/api/translating.rs b/src/api/translating.rs
@@ -74,6 +74,8 @@ pub async fn get(
     })))
 }
 
+const IGNORE_LANGGUAGES: [&'static str; 5] = ["abk", "ava", "bak", "lim", "nya"];
+
 pub async fn list_languages(
     to: PackObject<()>,
     State(_): State<Arc<AppState>>,
@@ -85,11 +87,13 @@ pub async fn list_languages(
             continue;
         }
 
-        list.push((
-            lg.to_639_3().to_string(),
-            lg.to_name().to_string(),
-            lg.to_autonym().unwrap().to_string(),
-        ));
+        if !IGNORE_LANGGUAGES.contains(&lg.to_639_3()) {
+            list.push((
+                lg.to_639_3().to_string(),
+                lg.to_name().to_string(),
+                lg.to_autonym().unwrap().to_string(),
+            ));
+        }
     }
     Ok(to.with(SuccessResponse {
         total_size: Some(list.len() as u64),
@@ -263,7 +267,7 @@ async fn translate(
                     rid = &rid,
                     gid = gid.to_string(),
                     cid = cid.to_string(),
-                    language = language.to_string(),
+                    language = target_lang.to_639_3().to_string(),
                     version = version,
                     elapsed = ai_elapsed;
                     "{}", err.to_string(),
@@ -276,7 +280,7 @@ async fn translate(
                     rid = &rid,
                     gid = gid.to_string(),
                     cid = cid.to_string(),
-                    language = language.to_string(),
+                    language = target_lang.to_639_3().to_string(),
                     version = version,
                     elapsed = ai_elapsed;
                     "success",
@@ -298,7 +302,7 @@ async fn translate(
             rid = &rid,
             gid = gid.to_string(),
             cid = cid.to_string(),
-            language = language.to_string(),
+            language = target_lang.to_639_3().to_string(),
             version = version,
             elapsed = start.elapsed().as_millis() as u64,
             pieces = pieces;
@@ -319,7 +323,7 @@ async fn translate(
                 rid = &rid,
                 gid = gid.to_string(),
                 cid = cid.to_string(),
-                language = language.to_string(),
+                language = target_lang.to_639_3().to_string(),
                 version = version,
                 elapsed = start.elapsed().as_millis() as u64,
                 pieces = pieces;
@@ -332,7 +336,7 @@ async fn translate(
                 rid = &rid,
                 gid = gid.to_string(),
                 cid = cid.to_string(),
-                language = language.to_string(),
+                language = target_lang.to_639_3().to_string(),
                 version = version,
                 elapsed = start.elapsed().as_millis() as u64,
                 pieces = pieces;

diff --git a/src/db/qdrant.rs b/src/db/qdrant.rs
@@ -2,9 +2,9 @@ use qdrant_client::client::{QdrantClient, QdrantClientConfig};
 use tokio::time::Duration;
 
 pub use qdrant_client::qdrant::{
-    r#match::MatchValue, Condition, FieldCondition, Filter, Match, PointId, PointStruct,
-    ReadConsistency, SearchPoints, SearchResponse, Value, Vectors, WithPayloadSelector,
-    WithVectorsSelector,
+    r#match::MatchValue, read_consistency, Condition, FieldCondition, Filter, Match, PointId,
+    PointStruct, ReadConsistency, SearchPoints, SearchResponse, Value, Vectors,
+    WithPayloadSelector, WithVectorsSelector,
 };
 
 use crate::conf;
@@ -64,7 +64,9 @@ impl Qdrant {
                 &ids,
                 Some(WithVectorsSelector::from(true)),
                 Some(WithPayloadSelector::from(true)),
-                Some(ReadConsistency::default()),
+                Some(ReadConsistency {
+                    value: Some(read_consistency::Value::Type(1)),
+                }),
             )
             .await?;
 

diff --git a/src/openai.rs b/src/openai.rs
@@ -167,6 +167,7 @@ impl OpenAI {
                         rid = rid,
                         user = user,
                         fixed = content.is_ok(),
+                        input = text,
                         output = output;
                         "",
                     );
@@ -177,6 +178,7 @@ impl OpenAI {
                         rid = rid,
                         user = user,
                         fixed = false,
+                        input = text,
                         output = choice.message.content;
                         "{}", &er,
                     );
@@ -214,6 +216,7 @@ impl OpenAI {
                 elapsed = start.elapsed().as_millis() as u64,
                 rid = rid,
                 user = user,
+                input = text,
                 output = choice.message.content;
                 "{}", err,
             );