From 26e4e3c686cf749ee9afc2dfd1ae908deafecc4f Mon Sep 17 00:00:00 2001 From: saattrupdan Date: Tue, 11 Jun 2024 15:54:19 +0200 Subject: [PATCH] tests: Use HUGGINGFACE_API_TOKEN --- .github/workflows/tests.yml | 2 ++ tests/fsm/test_regex.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 10879c78f..0384d1b63 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,8 @@ jobs: - name: Run tests run: | pytest --cov=outlines + env: + HUGGINGFACE_API_TOKEN: ${{ secrets.HUGGINGFACE_API_TOKEN }} - name: Upload coverage data uses: actions/upload-artifact@v3 with: diff --git a/tests/fsm/test_regex.py b/tests/fsm/test_regex.py index dffa37975..c716080dc 100644 --- a/tests/fsm/test_regex.py +++ b/tests/fsm/test_regex.py @@ -3,6 +3,7 @@ import numpy as np import pytest from transformers import AutoTokenizer +import os from outlines.fsm.regex import ( _walk_fsm, @@ -706,6 +707,7 @@ def test_numba_leading_null_byte_unicode_type_sane(input_key): ) def test_reduced_vocabulary_with_rare_tokens(model_id): """Assert reduced_vocabulary works with rare tokens.""" - tokenizer = AutoTokenizer.from_pretrained(model_id) + token = os.getenv("HUGGINGFACE_API_TOKEN") + tokenizer = AutoTokenizer.from_pretrained(model_id, token=token) tokenizer = adapt_tokenizer(tokenizer=tokenizer) reduced_vocabulary(tokenizer)