New dict

adamjgrant · Apr 30, 2024 · c584d15 · c584d15
1 parent e29e383
commit c584d15
Show file tree

Hide file tree

Showing 5 changed files with 2 additions and 2 deletions.
diff --git a/backup/dictionary.pkl b/backup/dictionary.pkl
diff --git a/backup/processing_progress.txt b/backup/processing_progress.txt
@@ -1 +1 @@
-25571507,23599999999
+26134921,24119999999
diff --git a/dictionary.msgpack b/dictionary.msgpack
diff --git a/tokens.msgpack b/tokens.msgpack
diff --git a/train.py b/train.py
@@ -105,7 +105,7 @@ async def main(retain=False):
               tree_store = await save_position('training/processing_progress.txt', i + start_position + 1, word_count, tree_store)
               gc.collect()
 
-          if (word_count + 1) % (PRUNE_FREQUENCY * 20) == 0:
+          if (word_count + 1) % (PRUNE_FREQUENCY * 10) == 0:
               merge_batches()
 
   await create_batch(tree_store, TARGET_DICTIONARY_COUNT)