Basic coverage for ModularTokenizerOp (#388)

* done after hooks * forgot the test lol <3 * add missing req * another missing req * another missing reqgit add -u! --------- Co-authored-by: Sagi Polaczek <[email protected]>
BiomedSciAI · Dec 19, 2024 · 238f84f · 238f84f
1 parent 804c185
commit 238f84f
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 1 deletion.
diff --git a/fuse/data/tests/test_initialization.py b/fuse/data/tests/test_initialization.py
@@ -0,0 +1,14 @@
+import unittest
+
+from fuse.data.tokenizers.modular_tokenizer.op import ModularTokenizerOp
+
+
+class TestInitialization(unittest.TestCase):
+    def test_from_pretrained(self) -> None:
+        tokenizer_op = ModularTokenizerOp.from_pretrained(
+            "ibm/biomed.omics.bl.sm.ma-ted-458m"
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/fuse/data/tokenizers/modular_tokenizer/op.py b/fuse/data/tokenizers/modular_tokenizer/op.py
@@ -8,11 +8,12 @@
 from huggingface_hub import HfApi, snapshot_download
 from huggingface_hub.utils import SoftTemporaryDirectory, validate_hf_hub_args
 
-from fuse.data import OpBase, get_sample_id
+from fuse.data import OpBase
 from fuse.data.tokenizers.modular_tokenizer.inject_utils import (
     InjectorToModularTokenizerLib,
 )
 from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ModularTokenizer
+from fuse.data.utils.sample import get_sample_id
 from fuse.utils import NDict
 
 

diff --git a/fuse/requirements.txt b/fuse/requirements.txt
@@ -32,3 +32,6 @@ clearml
 x-transformers
 jsonargparse
 click
+huggingface_hub
+tokenizers
+transformers