1. Fix #119 for OOV token update

2. Fix #118 for hyperpara tuning
reczoo · Oct 23, 2024 · b026ce9 · b026ce9
1 parent 9758a89
commit b026ce9
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -151,7 +151,7 @@ Please install other required packages via `pip install -r requirements.txt`.
 
    ```
    cd experiment
-   python run_param_tuner.py --config config/DCN_tiny_h5_tuner_config.yaml --gpu 0 1 2 3 0 1 2 3
+   python run_param_tuner.py --config config/DCN_tiny_parquet_tuner_config.yaml --gpu 0 1 2 3 0 1 2 3
    ```
 
 ## 🔥 Citation

diff --git a/...ent/config/DCN_tiny_npz_tuner_config.yaml → ...config/DCN_tiny_parquet_tuner_config.yaml b/...ent/config/DCN_tiny_npz_tuner_config.yaml → ...config/DCN_tiny_parquet_tuner_config.yaml
@@ -1,6 +1,13 @@
 base_config: ../model_zoo/DCN/DCN_torch/config/
 base_expid: DCN_default
-dataset_id: tiny_npz
+dataset_id: tiny_parquet
+
+tiny_parquet:
+    data_root: ../data/
+    data_format: npz
+    train_data: ../data/tiny_parquet/train.parquet
+    valid_data: ../data/tiny_parquet/valid.parquet
+    test_data: ../data/tiny_parquet/test.parquet
 
 tuner_space:
     model_root: './checkpoints/'

diff --git a/fuxictr/preprocess/tokenizer.py b/fuxictr/preprocess/tokenizer.py
@@ -86,7 +86,8 @@ def merge_vocab(self, shared_tokenizer):
         else:
             shared_tokenizer.vocab.update(self.vocab)
         vocab_size = shared_tokenizer.vocab_size()
-        if shared_tokenizer.vocab["__OOV__"] != vocab_size - 1:
+        if (shared_tokenizer.vocab["__OOV__"] != vocab_size - 1 or
+            shared_tokenizer.vocab["__OOV__"] != len(shared_tokenizer.vocab) - 1):
             shared_tokenizer.vocab["__OOV__"] = vocab_size
         self.vocab = shared_tokenizer.vocab
         return shared_tokenizer