From bc3c154c9658fc57a8abdf8a097b53826d0c8400 Mon Sep 17 00:00:00 2001
From: kingbri <8082010+bdashore3@users.noreply.github.com>
Date: Fri, 13 Dec 2024 00:58:25 -0500
Subject: [PATCH] Dependencies: Pin tokenizers

Use a version greater than 0.20.0 for newer model support.

Signed-off-by: kingbri <8082010+bdashore3@users.noreply.github.com>
---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 021b6a9..5d7acb7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
     "loguru",
     "sse-starlette",
     "packaging",
-    "tokenizers",
+    "tokenizers>=0.21.0",
     "formatron",
     "kbnf>=0.4.1",
     "aiofiles",