tests: Use HUGGINGFACE_API_TOKEN

dottxt-ai · Jun 11, 2024 · 26e4e3c · 26e4e3c
1 parent ef03abb
commit 26e4e3c
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -36,6 +36,8 @@ jobs:
     - name: Run tests
       run: |
         pytest --cov=outlines
+      env:
+        HUGGINGFACE_API_TOKEN: ${{ secrets.HUGGINGFACE_API_TOKEN }}
     - name: Upload coverage data
       uses: actions/upload-artifact@v3
       with:

diff --git a/tests/fsm/test_regex.py b/tests/fsm/test_regex.py
@@ -3,6 +3,7 @@
 import numpy as np
 import pytest
 from transformers import AutoTokenizer
+import os
 
 from outlines.fsm.regex import (
     _walk_fsm,
@@ -706,6 +707,7 @@ def test_numba_leading_null_byte_unicode_type_sane(input_key):
 )
 def test_reduced_vocabulary_with_rare_tokens(model_id):
     """Assert reduced_vocabulary works with rare tokens."""
-    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    token = os.getenv("HUGGINGFACE_API_TOKEN")
+    tokenizer = AutoTokenizer.from_pretrained(model_id, token=token)
     tokenizer = adapt_tokenizer(tokenizer=tokenizer)
     reduced_vocabulary(tokenizer)