placeholder token: "░" -> " " (for mistral)

microsoft · Jan 26, 2024 · 3ceb583 · 3ceb583
1 parent 545d611
commit 3ceb583
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/declctrl/src/declctrl.rs b/declctrl/src/declctrl.rs
@@ -1259,7 +1259,7 @@ impl AiciCtrl for Runner {
             // this is a bit late in the game, but it's the best we can do
             MidProcessResult::Splice {
                 backtrack: 0,
-                ff_tokens: tokenize("░"),
+                ff_tokens: tokenize(" "),
             }
 
             // // we pop the useless generated token

diff --git a/jsctrl/ts/aici.ts b/jsctrl/ts/aici.ts
@@ -562,7 +562,7 @@ export class AiciAsync implements AiciCallbacks {
       if (r2._n_suspended) {
         // Need to generate one fake token...
         this._pendingCb = this._token;
-        const f = new FixedTokens("░");
+        const f = new FixedTokens(" ");
         assert(f.fixedTokens.length === 1);
         this._token = f;
       }

diff --git a/pyaici/server.py b/pyaici/server.py
@@ -396,7 +396,7 @@ def mid_process(self, fork_group: list[SeqId]) -> MidProcessResult:
             if r2.suspended:
                 # need to generate one fake token...
                 self._pending_cb = self._cb
-                f = FixedTokens("░")
+                f = FixedTokens(" ")
                 assert len(f.fixed_tokens) == 1
                 self._cb = f
             r = self._cb._mid_process(fork_group)