Add tokenizer & model loading

tests/web.rs

@@ -3,8 +3,10 @@
 #![cfg(target_arch = "wasm32")]
 
 extern crate wasm_bindgen_test;
+use std::fmt::format;
 use std::println;
 
+use js_sys::Uint8Array;
 use wasm_bindgen::{prelude::*, JsValue};
 use wasm_bindgen_futures::JsFuture;
 use wasm_bindgen_test::*;
@@ -13,7 +15,7 @@ use web_sys::{Request, RequestInit, RequestMode, Response};
 
 wasm_bindgen_test_configure!(run_in_browser);
 
-async fn load_json(url: &str) -> Result<JsValue, JsValue> {
+async fn fetch(url: &str) -> Result<Response, JsValue> {
     let mut opts = RequestInit::new();
     opts.method("GET");
     opts.mode(RequestMode::Cors);
@@ -29,20 +31,36 @@ async fn load_json(url: &str) -> Result<JsValue, JsValue> {
     assert!(resp_value.is_instance_of::<Response>());
     let resp: Response = resp_value.dyn_into().unwrap();
 
-    // Convert this other `Promise` into a rust `Future`.
-    let json = JsFuture::from(resp.json()?).await?;
+    Ok(resp)
+}
+
+async fn load_json(url: &str) -> Result<JsValue, JsValue> {
+    let response = fetch(url).await?;
+    let json = JsFuture::from(response.json()?).await?;
 
     Ok(json)
 }
+
+async fn load_binary(url: &str) -> Result<Vec<u8>, JsValue> {
+    let response = fetch(url).await?;
+    let ab = JsFuture::from(response.array_buffer()?).await?;
+    let vec = Uint8Array::new(&ab).to_vec();
+    Ok(vec)
+}
+
 #[wasm_bindgen_test]
 async fn pass() -> Result<(), JsValue> {
-    let url = "http://localhost:45678/tokenizer.json";
+    let tokenizer_url = "http://localhost:45678/tokenizer.json";
+    let model_url = "http://localhost:45678/tinymistral-248m.q4_k_m.gguf";
 
-    let json = load_json(&url).await?;
+    let tokenizer: Vec<u8> = load_binary(&tokenizer_url).await?;
+    let tokenizer_len = format!("{}", &tokenizer.len());
+    console::log_2(&"tokenizer size".into(), &tokenizer_len.into());
 
-    let json_str = js_sys::JSON::stringify(&json)?;
+    let model: Vec<u8> = load_binary(&model_url).await?;
+    let model_len = format!("{}", &model.len());
+    console::log_2(&"model size".into(), &model_len.into());
 
-    console::log_2(&"Logging arbitrary values looks like".into(), &json_str);
     assert_eq!(1 + 1, 2);
     Ok(())
 }