Dependencies: Pin tokenizers

Use a version greater than 0.20.0 for newer model support. Signed-off-by: kingbri <[email protected]>
theroyallab · Dec 13, 2024 · bc3c154 · bc3c154
1 parent 1ba33bf
commit bc3c154
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/pyproject.toml b/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
     "loguru",
     "sse-starlette",
     "packaging",
-    "tokenizers",
+    "tokenizers>=0.21.0",
     "formatron",
     "kbnf>=0.4.1",
     "aiofiles",