diff --git a/yomikomi/src/strided_index.rs b/yomikomi/src/strided_index.rs index d7ae08b..1aff318 100644 --- a/yomikomi/src/strided_index.rs +++ b/yomikomi/src/strided_index.rs @@ -27,7 +27,7 @@ impl<'a> StridedIndex<'a> { } } -impl<'a> Iterator for StridedIndex<'a> { +impl Iterator for StridedIndex<'_> { type Item = usize; fn next(&mut self) -> Option { diff --git a/yomikomi/src/tokenize.rs b/yomikomi/src/tokenize.rs index 1ae1951..069392f 100644 --- a/yomikomi/src/tokenize.rs +++ b/yomikomi/src/tokenize.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex}; use tokenizers::tokenizer::Tokenizer; enum Processor { - Tokenizers { inner: Tokenizer, bos_id: Option, eos_id: Option }, + Tokenizers { inner: Box, bos_id: Option, eos_id: Option }, SentencePiece(SentencePieceProcessor), } @@ -59,7 +59,7 @@ impl Tokenize { ) -> Result { let path = path.as_ref(); let processor = if path.extension().map_or(false, |v| v == "json") { - let inner = Tokenizer::from_file(path)?; + let inner = Box::new(Tokenizer::from_file(path)?); Processor::Tokenizers { inner, bos_id: None, eos_id: None } } else { Processor::SentencePiece(SentencePieceProcessor::open(path).map_err(E::wrap)?)