Draft tokenizer update

triton-inference-server · Dec 3, 2024 · 344cfc6 · 344cfc6
1 parent aa1c872
commit 344cfc6
Show file tree

Hide file tree

Showing 5 changed files with 2,062 additions and 186 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -53,7 +53,7 @@ repos:
   - id: codespell
     additional_dependencies: [tomli]
     args: ["--toml", "pyproject.toml"]
-    exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$)
+    exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$|.*sonnets.txt)
 # More details about these pre-commit hooks here:
 # https://pre-commit.com/hooks.html
 - repo: https://github.com/pre-commit/pre-commit-hooks

diff --git a/genai-perf/genai_perf/inputs/retrievers/farewell.txt b/genai-perf/genai_perf/inputs/retrievers/farewell.txt