Sync codebase

Author: hauntsaninja

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tiktoken"
-version = "0.9.0"
+version = "0.10.0"
 edition = "2021"
 rust-version = "1.57.0"
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tiktoken"
-version = "0.9.0"
+version = "0.10.0"
 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
 readme = "README.md"
 license = { file = "LICENSE" }

tiktoken/__init__.py

@@ -5,4 +5,4 @@
 from .registry import get_encoding as get_encoding
 from .registry import list_encoding_names as list_encoding_names
 
-__version__ = "0.9.0"
+__version__ = "0.10.0"

tiktoken/model.py

@@ -7,12 +7,16 @@
 MODEL_PREFIX_TO_ENCODING: dict[str, str] = {
     "o1-": "o200k_base",
     "o3-": "o200k_base",
+    "o4-mini-": "o200k_base",
     # chat
+    "gpt-4.5-": "o200k_base",
+    "gpt-4.1-": "o200k_base",
     "chatgpt-4o-": "o200k_base",
     "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13
     "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
     "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
     "gpt-35-turbo-": "cl100k_base",  # Azure deployment name
+    "gpt-oss-": "o200k_harmony",
     # fine-tuned
     "ft:gpt-4o": "o200k_base",
     "ft:gpt-4": "cl100k_base",
@@ -25,7 +29,9 @@
     # reasoning
     "o1": "o200k_base",
     "o3": "o200k_base",
+    "o4-mini": "o200k_base",
     # chat
+    "gpt-4.1": "o200k_base",
     "gpt-4o": "o200k_base",
     "gpt-4": "cl100k_base",
     "gpt-3.5-turbo": "cl100k_base",

tiktoken_ext/openai_public.py

@@ -120,11 +120,43 @@ def o200k_base():
     }
 
 
+def o200k_harmony():
+    base_enc = o200k_base()
+    name = "o200k_harmony"
+    pat_str = base_enc["pat_str"]
+    mergeable_ranks = base_enc["mergeable_ranks"]
+    special_tokens = {
+        **base_enc["special_tokens"],
+        "<|startoftext|>": 199998,
+        "<|endoftext|>": 199999,
+        "<|reserved_200000|>": 200000,
+        "<|reserved_200001|>": 200001,
+        "<|return|>": 200002,
+        "<|constrain|>": 200003,
+        "<|reserved_200004|>": 200004,
+        "<|channel|>": 200005,
+        "<|start|>": 200006,
+        "<|end|>": 200007,
+        "<|message|>": 200008,
+        "<|reserved_200009|>": 200009,
+        "<|reserved_200010|>": 200010,
+        "<|reserved_200011|>": 200011,
+        "<|call|>": 200012,
+    } | {f"<|reserved_{i}|>": i for i in range(200013, 201088)}
+    return {
+        "name": name,
+        "pat_str": pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
 ENCODING_CONSTRUCTORS = {
     "gpt2": gpt2,
     "r50k_base": r50k_base,
     "p50k_base": p50k_base,
     "p50k_edit": p50k_edit,
     "cl100k_base": cl100k_base,
     "o200k_base": o200k_base,
+    "o200k_harmony": o200k_harmony,
 }