diff --git a/src/lib.rs b/src/lib.rs index 2cba82ad..85cd301b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,6 @@ pub mod regex; pub mod vocabulary; mod locator; -mod processor; #[cfg(feature = "python-bindings")] mod python_bindings; diff --git a/src/vocabulary.rs b/src/vocabulary/mod.rs similarity index 99% rename from src/vocabulary.rs rename to src/vocabulary/mod.rs index 438e94b5..7972ab3c 100644 --- a/src/vocabulary.rs +++ b/src/vocabulary/mod.rs @@ -5,9 +5,12 @@ use tokenizers::{FromPretrainedParameters, NormalizerWrapper, Tokenizer}; use crate::locator::EosTokenLocator; use crate::prelude::*; -use crate::processor::TokenProcessor; use crate::VocabularyError; +use processor::TokenProcessor; + +mod processor; + /// Vocabulary of an LLM. /// /// ## Examples diff --git a/src/processor.rs b/src/vocabulary/processor.rs similarity index 100% rename from src/processor.rs rename to src/vocabulary/processor.rs