Merge branch 'main' of https://github.com/microsoft/gvm

.gitmodules

@@ -8,3 +8,6 @@
 [submodule "llama-cpp-low/llama.cpp"]
 	path = rllm/llama-cpp-low/llama.cpp
 	url = https://github.com/ggerganov/llama.cpp
+[submodule "py/guidance"]
+	path = py/guidance
+	url = https://github.com/guidance-ai/guidance

.vscode/settings.json

@@ -54,7 +54,8 @@
         "regex": "cpp"
     },
     "python.analysis.extraPaths": [
-        "./vllm",
+        "./py/vllm",
+        "./py/guidance",
         "./py"
     ],
     "spellright.language": [
@@ -105,6 +106,7 @@
         "NOSYS",
         "nvprof",
         "POSIX",
+        "protobuf",
         "pyaici",
         "pyctrl",
         "pylib",

NOTICE.md

@@ -544,31 +544,3 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ```
-
-## Component: Guidance
-
-**Open Source License**:
-
-```
-MIT License
-
-Copyright (c) The Guidance Contributors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-```

README.md

@@ -427,8 +427,6 @@ All of these can be easily extended.
 - specific [Python library](./controllers/pyctrl/Lib/) files are copied from
   [RustPython](https://github.com/RustPython/RustPython)
   (as we only use a subset of them)
-- [Guidance](https://github.com/guidance-ai/guidance) files copied under
-  [guidance_ctrl](./controllers/guidance_ctrl/Lib/guidance/)
 - the [example ANSI C grammar](controllers/aici_abi/grammars/c.y) is based on
   https://www.lysator.liu.se/c/ANSI-C-grammar-y.html by Jeff Lee (from 1985)
 

aicirt/src/bintokens.rs

@@ -11,9 +11,8 @@ pub struct ByteTokenizer {
     pub hf_tokenizer: Tokenizer,
     pub eos_token: u32,
     pub vocab_size: u32,
+    token_bytes: Vec<Vec<u8>>,
     pub special: BTreeMap<String, u32>,
-    pub binary: BTreeMap<String, u32>,
-    pub text: BTreeMap<String, u32>,
 }
 
 pub struct TokenizerInfo {
@@ -187,14 +186,6 @@ pub fn find_tokenizer(mut name: &str) -> Result<ByteTokenizer> {
     }
 }
 
-fn from_hex(hex_str: &str) -> Result<Vec<u8>> {
-    let mut bytes = Vec::new();
-    for i in (0..hex_str.len()).step_by(2) {
-        bytes.push(u8::from_str_radix(&hex_str[i..(i + 2)], 16)?);
-    }
-    Ok(bytes)
-}
-
 impl ByteTokenizer {
     pub fn from_tokenizer(mut hft: Tokenizer) -> Result<ByteTokenizer> {
         let mut is_byte_level = false;
@@ -256,8 +247,7 @@ impl ByteTokenizer {
             eos_token: 0,
             vocab_size,
             special: BTreeMap::new(),
-            binary: BTreeMap::new(),
-            text: BTreeMap::new(),
+            token_bytes: (0..vocab_size).map(|_| Vec::new()).collect(),
             hf_tokenizer: hft,
         };
 
@@ -269,7 +259,7 @@ impl ByteTokenizer {
                 }
                 res.special.insert(info.content.clone(), *id);
             } else {
-                res.text.insert(info.content.clone(), *id);
+                res.token_bytes[*id as usize] = info.content.clone().into_bytes();
             }
         }
 
@@ -286,17 +276,11 @@ impl ByteTokenizer {
                         // parse hex number from tok_name
                         let hex_str = &tok_name[3..5];
                         let byte = u8::from_str_radix(hex_str, 16).unwrap();
-                        if byte >= 0x80 {
-                            let s = format!("{:02x}", byte);
-                            res.binary.insert(s, tok_id);
-                        } else {
-                            let s = format!("{}", byte as char);
-                            res.text.insert(s, tok_id);
-                        }
+                        res.token_bytes[tok_id as usize] = vec![byte];
                     } else {
                         assert!(!tok_name.starts_with("<0x"));
                         let tok_name = tok_name.replace(space_ch, " ");
-                        res.text.insert(tok_name, tok_id);
+                        res.token_bytes[tok_id as usize] = tok_name.as_bytes().to_vec();
                     }
                 } else if is_byte_level {
                     let bytes: Result<Vec<u8>> = tok_name
@@ -316,12 +300,7 @@ impl ByteTokenizer {
                         }
                     };
 
-                    if let Ok(s) = String::from_utf8(bytes.clone()) {
-                        res.text.insert(s, tok_id);
-                    } else {
-                        let hexstr = String::from_iter(bytes.iter().map(|b| format!("{:02x}", b)));
-                        res.binary.insert(hexstr, tok_id);
-                    }
+                    res.token_bytes[tok_id as usize] = bytes;
                 } else {
                     panic!();
                 }
@@ -342,22 +321,6 @@ impl ByteTokenizer {
         }
     }
     pub fn token_bytes(&self) -> Vec<Vec<u8>> {
-        let tinfo = self.tokrx_info();
-        let mut r = Vec::with_capacity(tinfo.vocab_size as usize);
-        r.resize_with(tinfo.vocab_size as usize, Vec::new);
-
-        let info = self;
-
-        for (k, v) in &info.text {
-            let idx = *v as usize;
-            r[idx] = k.as_bytes().to_vec();
-        }
-
-        for (k, v) in &info.binary {
-            let idx = *v as usize;
-            r[idx] = from_hex(k).unwrap();
-        }
-
-        r
+        self.token_bytes.clone()
     }
 }

aicirt/src/futexshm.rs

@@ -3,10 +3,7 @@ use anyhow::{anyhow, Result};
 use serde::{Deserialize, Serialize};
 use std::{
     ptr,
-    sync::{
-        atomic::{AtomicU32, Ordering},
-        Arc,
-    },
+    sync::atomic::{AtomicU32, Ordering},
     time::{Duration, Instant},
 };
 
@@ -25,53 +22,6 @@ fn futex_at(shm: &Shm, off: usize) -> &'static Futex {
     unsafe { AtomicU32::from_ptr(shm.ptr_at(off) as *mut u32).as_futex() }
 }
 
-pub struct WrMsgCounter {
-    futex: &'static Futex,
-    #[allow(dead_code)]
-    shm: Shm,
-}
-
-impl WrMsgCounter {
-    pub fn new(shm: Shm) -> Self {
-        Self {
-            futex: futex_at(&shm, 0),
-            shm,
-        }
-    }
-
-    pub fn get(&self) -> u32 {
-        self.futex.value.load(Ordering::Acquire)
-    }
-
-    pub fn inc(&self) {
-        let _ = self.futex.value.fetch_add(1, Ordering::AcqRel);
-        let _ = self.futex.wake(i32::MAX);
-    }
-}
-
-pub struct RdMsgCounter {
-    futex: &'static Futex,
-    #[allow(dead_code)]
-    shm: Arc<Shm>,
-}
-
-impl RdMsgCounter {
-    pub fn new(shm: Arc<Shm>) -> Self {
-        Self {
-            futex: futex_at(&shm, 0),
-            shm,
-        }
-    }
-
-    pub fn read(&self) -> u32 {
-        self.futex.value.load(Ordering::Acquire)
-    }
-
-    pub fn wait(&self, val: u32) {
-        let _ = self.futex.wait(val);
-    }
-}
-
 struct Channel {
     wr_len: &'static Futex,
     rd_len: &'static Futex,
@@ -250,15 +200,12 @@ impl ClientChannel {
 
 pub struct ServerChannel {
     channel: Channel,
-    #[allow(dead_code)]
-    msg_cnt: RdMsgCounter,
 }
 
 impl ServerChannel {
-    pub fn new(shm: Shm, cnt_shm: Arc<Shm>) -> Self {
+    pub fn new(shm: Shm) -> Self {
         Self {
             channel: Channel::new(shm, true),
-            msg_cnt: RdMsgCounter::new(cnt_shm),
         }
     }
 
@@ -296,9 +243,9 @@ where
     Cmd: for<'d> Deserialize<'d> + Serialize,
     Resp: for<'d> Deserialize<'d> + Serialize,
 {
-    pub fn new(shm: Shm, cnt_shm: Arc<Shm>) -> Self {
+    pub fn new(shm: Shm) -> Self {
         Self {
-            channel: ServerChannel::new(shm, cnt_shm),
+            channel: ServerChannel::new(shm),
             _cmd: std::marker::PhantomData,
             _resp: std::marker::PhantomData,
         }

aicirt/src/hostimpl.rs

@@ -8,14 +8,18 @@ use aici_abi::{
 };
 use aicirt::user_error;
 use anyhow::{anyhow, Result};
-use std::{rc::Rc, sync::Arc, time::Duration};
+use std::{
+    rc::Rc,
+    sync::Arc,
+    time::{Duration, Instant},
+};
 use tokenizers::Tokenizer;
 
 #[derive(Clone)]
 pub struct AiciLimits {
     pub ipc_shm_bytes: usize,
-    pub msg_cnt: Arc<Shm>,
 
+    pub timer_resolution_ns: u64,
     pub max_memory_bytes: usize,
     pub max_pre_step_ms: u64,
     pub max_step_ms: u64,
@@ -37,13 +41,15 @@ pub struct ModuleData {
     pub group_channel: GroupHandle,
     pub process_result: Vec<u8>,
     pub logit_ptr: &'static mut [f32],
+    pub limits: AiciLimits,
     pub linker: Arc<wasmtime::Linker<ModuleData>>,
     pub instance: Option<wasmtime::Instance>,
     pub memory: Option<wasmtime::Memory>,
     pub module: wasmtime::Module,
     pub store_limits: wasmtime::StoreLimits,
     pub had_error: bool,
     pub storage_log: Vec<StorageCmd>,
+    pub start_time: Instant,
     blobs: Vec<Rc<Vec<u8>>>,
 }
 
@@ -92,6 +98,7 @@ impl ModuleData {
             globals,
             group_channel,
             module: module.clone(),
+            limits: limits.clone(),
             linker: linker.clone(),
             instance: None,
             memory: None,
@@ -100,6 +107,7 @@ impl ModuleData {
             logit_ptr: &mut [],
             had_error: false,
             storage_log: Vec::new(),
+            start_time: Instant::now(),
             blobs: vec![Rc::new(Vec::new()); BlobId::MAX_BLOB_ID as usize],
         };
         r.set_blob(BlobId::MODULE_ARG, module_arg.as_bytes().to_vec());
@@ -290,7 +298,6 @@ pub fn setup_linker(engine: &wasmtime::Engine) -> Result<Arc<wasmtime::Linker<Mo
     let mut linker = wasmtime::Linker::<ModuleData>::new(engine);
 
     fake_wasi!(linker, environ_get, i32 i32);
-    fake_wasi!(linker, clock_time_get, i32 i64 i32);
     fake_wasi!(linker, path_create_directory, i32 i32 i32);
     fake_wasi!(linker, path_filestat_get, i32 i32 i32 i32 i32);
     fake_wasi!(linker, path_link, i32 i32 i32 i32 i32 i32 i32);
@@ -334,6 +341,27 @@ pub fn setup_linker(engine: &wasmtime::Engine) -> Result<Arc<wasmtime::Linker<Mo
         },
     )?;
 
+    linker.func_wrap(
+        "wasi_snapshot_preview1",
+        "clock_time_get",
+        |mut caller: wasmtime::Caller<'_, ModuleData>,
+         clock_id: i32,
+         _precision: i64,
+         dst_ptr: u32|
+         -> Result<i32> {
+            if clock_id != 1 {
+                return Ok(63); // EPERM
+            }
+            let res = caller.data().limits.timer_resolution_ns as u64;
+            let now = std::time::Instant::now();
+            let nanos = now.duration_since(caller.data().start_time).as_nanos() as u64;
+            let nanos = if res == 0 { 0 } else { nanos / res * res };
+            let bytes = nanos.to_le_bytes();
+            write_caller_mem(&mut caller, dst_ptr, 8, &bytes);
+            Ok(0)
+        },
+    )?;
+
     linker.func_wrap(
         "wasi_snapshot_preview1",
         "fd_write",