diff --git a/CHANGELOG.md b/CHANGELOG.md
index e9752a733..26b30bb52 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added `OLMoE`: Configurations & modeling for training Mixture-of-Experts models.
 - Added ability to try loading latest checkpoint from save folder using `--try_load_latest_save`.
 - Added support for flash attention and gradient checkpointing to `hf_olmo`.
 
diff --git a/README.md b/README.md
index 095f35a13..2db12cb5f 100644
--- a/README.md
+++ b/README.md
@@ -221,3 +221,15 @@ See [Debugging](https://github.com/allenai/OLMo/blob/main/docs/NOTES.md#debuggin
   journal={arXiv preprint},
 }
 ```
+
+```bibtex
+@misc{muennighoff2024olmoeopenmixtureofexpertslanguage,
+      title={OLMoE: Open Mixture-of-Experts Language Models}, 
+      author={Niklas Muennighoff and Luca Soldaini and Dirk Groeneveld and Kyle Lo and Jacob Morrison and Sewon Min and Weijia Shi and Pete Walsh and Oyvind Tafjord and Nathan Lambert and Yuling Gu and Shane Arora and Akshita Bhagia and Dustin Schwenk and David Wadden and Alexander Wettig and Binyuan Hui and Tim Dettmers and Douwe Kiela and Ali Farhadi and Noah A. Smith and Pang Wei Koh and Amanpreet Singh and Hannaneh Hajishirzi},
+      year={2024},
+      eprint={2409.02060},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2409.02060}, 
+}
+```
diff --git a/configs/official/OLMoE-7B-A1B.yaml b/configs/official/OLMoE-7B-A1B.yaml
new file mode 100644
index 000000000..9927582cf
--- /dev/null
+++ b/configs/official/OLMoE-7B-A1B.yaml
@@ -0,0 +1,1493 @@
+run_name: olmoe
+seed: 6198
+dry_run: false
+no_pre_train_checkpoint: true
+
+wandb:
+  name: ${run_name}
+  project: olmoe
+  group: null
+
+model:
+  d_model: 2048
+  n_heads: 16
+  n_layers: 16
+  mlp_ratio: 1
+  weight_tying: false
+  alibi: false
+  rope: true
+  flash_attention: true
+  attention_dropout: 0.0
+  attention_layer_norm: true
+  include_bias: false
+  block_type: moe
+  layer_norm_type: rms
+  layer_norm_with_affine: true
+  bias_for_layer_norm: false
+  attention_layer_norm_with_affine: true
+  activation_type: swiglu
+  residual_dropout: 0.0
+  embedding_dropout: 0.0
+  max_sequence_length: 4096
+  vocab_size: 50280
+  embedding_size: 50304
+  eos_token_id: 0
+  pad_token_id: 1
+  init_device: meta
+  init_fn: normal
+  init_std: 0.02
+  init_cutoff_factor: 3
+  moe_top_k: 8
+  moe_num_experts: 64
+  moe_dropless: true
+  moe_mlp_impl: sparse
+  moe_zloss_weight: 0.001
+  moe_loss_weight: 0.01
+
+compile: null
+
+optimizer:
+  name: adamw
+  learning_rate: 4.0e-4
+  eps: 1.0e-8
+  weight_decay: 0.1
+  decay_norm_and_bias: true
+  decay_embeddings: true
+  betas:
+  - 0.9
+  - 0.95
+  metrics_log_interval: 10
+
+scheduler:
+  name: cosine_with_warmup
+  units: tokens
+  t_warmup: 10485760000
+  t_max: 5e12
+  alpha_f: 0.1
+
+tokenizer:
+  identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
+  truncate_direction: right
+
+save_folder: runs/${run_name}
+save_overwrite: false
+
+save_interval: 5000
+save_interval_ephemeral: null
+save_num_checkpoints_to_keep: -1
+sharded_checkpointer: olmo_core
+
+save_interval_unsharded: null
+save_num_unsharded_checkpoints_to_keep: -1
+
+load_path: null
+
+max_duration: 2ep
+global_train_batch_size: 1024
+device_train_microbatch_size: 4
+
+precision: amp_bf16
+distributed_strategy: fsdp
+
+fsdp:
+  wrapping_strategy: by_block
+  precision: mixed
+  sharding_strategy: FULL_SHARD
+
+max_grad_norm: 1.0
+max_grad_norm_ratio: null
+
+speed_monitor:
+  window_size: 1
+
+eval_interval: 5000
+eval_subset_num_batches: -1
+device_eval_batch_size: ${device_train_microbatch_size}
+evaluators:
+  - label: all-small-ppl-validation
+    data:
+      num_workers: 0
+      drop_last: true
+      datasets:
+        c4_en-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
+        dolma_books-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
+        dolma_common-crawl-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
+        dolma_pes2o-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
+        dolma_reddit-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
+        dolma_stack-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
+        dolma_wiki-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
+        ice-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
+        m2d2_s2orc-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
+        pile-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
+        wikitext_103-validation:
+          - /weka/oe-training-default/ai2-llm/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
+
+  ##########################
+  # Downstream evaluations #
+  ##########################
+  - label: piqa
+    type: downstream
+
+  - label: hellaswag
+    type: downstream
+
+  - label: winogrande
+    type: downstream
+
+  - label: openbook_qa
+    type: downstream
+
+  - label: boolq
+    type: downstream
+ 
+  - label: sciq
+    type: downstream
+
+  - label: arc_easy
+    type: downstream
+
+  - label: arc_challenge
+    type: downstream
+
+  - label: copa
+    type: downstream
+
+  - label: commonsense_qa
+    type: downstream
+
+  - label: social_iqa
+    type: downstream
+
+  - label: mmlu_stem_var
+    type: downstream
+
+  - label: mmlu_humanities_var
+    type: downstream
+
+  - label: mmlu_social_sciences_var
+    type: downstream
+
+  - label: mmlu_other_var
+    type: downstream
+
+  - label: mmlu_stem_mc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_mc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_mc_5shot
+    type: downstream
+
+  - label: mmlu_other_mc_5shot
+    type: downstream
+
+  - label: mmlu_stem_mc_5shot_test
+    type: downstream
+
+  - label: mmlu_humanities_mc_5shot_test
+    type: downstream
+
+  - label: mmlu_social_sciences_mc_5shot_test
+    type: downstream
+
+  - label: mmlu_other_mc_5shot_test
+    type: downstream
+
+data:
+  pad_direction: right
+  num_workers: 32
+  drop_last: true
+  pin_memory: true
+  prefetch_factor: 8
+  persistent_workers: true
+  timeout: 0
+  instance_filter:
+      repetition_max_period: 13
+      repetition_min_period: 1
+      repetition_max_count: 32  
+  paths:
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-014-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-015-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-016-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-017-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-018-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-019-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-020-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-021-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-022-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-023-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-024-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-025-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-026-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-027-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-028-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-029-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-030-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-031-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-032-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-033-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-034-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-035-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-036-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-037-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-038-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-039-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-040-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-041-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-042-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-043-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-044-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-045-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-046-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-047-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-048-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-049-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-050-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-051-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-052-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-053-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-054-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-055-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-056-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-057-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-058-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-059-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-060-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-061-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-062-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-063-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-064-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-065-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-066-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-067-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-068-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-069-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-070-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-071-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-072-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-073-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-074-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-075-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-076-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-077-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-078-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-079-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-080-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-081-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-082-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-083-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-084-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-085-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-086-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-087-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-088-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-089-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-090-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-091-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-092-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-093-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-094-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-095-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-096-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-097-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-098-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-099-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-16-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-17-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-18-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-19-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-20-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-20-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-21-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-21-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-22-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-23-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-23-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-24-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-24-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-25-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/pes2o/allenai/gpt-neox-olmo-dolma-v1_5/part-25-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-16-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-16-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-16-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-16-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-17-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-17-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-17-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-17-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-18-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-18-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-18-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-19-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-19-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-19-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-20-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-20-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-20-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-20-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-21-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-21-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-21-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-22-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-22-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-22-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-22-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-23-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-23-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-23-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-24-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-24-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-24-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-25-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-25-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-25-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-26-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-26-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-26-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-27-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-27-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-27-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-27-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-28-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-28-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-28-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-29-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-29-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-29-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-30-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-30-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-30-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-31-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-31-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-31-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-31-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-32-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-32-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-32-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-33-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-33-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-34-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-34-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-34-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-35-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-35-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-35-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-36-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-36-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-36-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-36-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-37-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-37-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-37-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-37-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-38-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-38-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-38-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-39-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-39-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-40-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-40-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-40-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-41-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-41-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-41-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-42-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-42-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-42-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-43-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-43-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-43-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-44-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-44-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-44-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-45-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-45-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-45-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-46-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-46-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-46-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-46-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-47-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-47-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-47-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-48-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-48-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-48-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/starcoder/allenai/gpt-neox-olmo-dolma-v1_5/part-48-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/wiki/allenai/gpt-neox-olmo-dolma-v1_5/part-0-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/wiki/allenai/gpt-neox-olmo-dolma-v1_5/part-0-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/wiki/allenai/gpt-neox-olmo-dolma-v1_5/part-0-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/wiki/allenai/gpt-neox-olmo-dolma-v1_5/part-0-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/wiki/allenai/gpt-neox-olmo-dolma-v1_5/part-1-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/danyh-compiled-v1_7/wiki/allenai/gpt-neox-olmo-dolma-v1_5/part-1-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-080-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-061-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-009-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-096-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-065-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-111-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-051-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-168-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-056-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-126-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-017-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-120-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-143-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-011-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-079-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-094-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-143-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-030-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-014-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-140-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-034-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-115-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-140-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-130-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-114-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-117-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-100-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-022-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-088-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-055-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-050-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-112-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-093-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-142-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-036-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-145-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-175-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-132-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-111-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-051-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-039-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-184-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-161-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-026-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-176-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-026-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-028-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-107-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-021-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-091-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-162-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-165-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-153-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-078-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-015-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-057-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-024-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-096-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-135-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-097-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-075-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-028-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-120-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-177-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-142-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-127-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-064-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-072-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-000-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-024-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-101-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-182-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-139-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-113-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-182-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-170-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-180-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-077-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-000-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-009-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-019-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-063-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-102-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-102-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-166-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-186-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-033-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-002-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-044-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-122-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-168-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-018-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-146-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-173-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-063-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-006-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-067-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-166-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-152-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-116-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-019-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-014-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-019-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-091-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-102-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-125-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-013-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-064-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-121-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-139-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-149-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-167-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-004-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-111-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-009-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-031-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-103-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-150-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-170-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-063-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-133-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-081-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-061-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-144-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-148-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-034-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-070-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-116-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-163-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-043-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-178-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-086-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-105-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-095-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-181-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-056-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-008-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-154-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-050-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-114-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-023-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-076-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-015-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-086-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-124-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-172-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-093-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-138-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-179-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-039-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-024-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-159-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-161-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-134-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-018-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-106-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-013-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-081-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-010-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-065-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-116-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-009-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-088-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-046-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-001-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-135-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-175-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-183-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-045-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-062-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-147-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-151-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-051-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-005-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-003-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-050-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-054-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-109-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-056-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-100-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-149-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-142-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-173-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-018-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-158-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-059-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-126-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-010-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-120-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-136-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-132-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-081-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-133-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-125-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-062-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-067-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-082-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-068-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-104-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-053-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-165-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-148-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-107-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-176-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-020-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-040-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-054-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-012-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-134-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-108-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-146-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-160-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-114-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-084-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-141-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-040-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-087-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-145-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-093-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-099-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-064-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-158-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-020-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-133-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-083-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-084-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-070-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-005-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-139-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-078-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-109-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-086-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-054-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-128-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-021-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-159-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-005-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-175-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-159-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-142-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-117-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-029-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-047-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-018-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-119-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-162-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-002-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-013-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-002-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-090-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-152-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-184-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-082-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-010-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-164-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-168-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-067-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-081-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-084-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-052-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-103-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-108-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-032-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-074-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-065-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-157-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-167-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-180-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-178-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-052-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-099-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-167-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-016-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-089-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-006-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-001-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-094-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-135-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-169-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-077-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-059-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-096-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-121-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-142-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-010-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-016-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-068-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-138-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-104-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-008-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-154-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-167-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-083-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-107-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-164-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-051-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-186-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-011-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-036-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-052-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-064-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-164-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-007-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-056-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-098-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-083-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-099-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-154-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-165-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-168-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-131-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-092-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-060-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-039-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-134-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-163-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-186-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-015-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-047-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-029-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-073-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-151-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-109-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-124-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-023-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-045-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-053-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-060-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-079-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-141-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-087-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-033-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-118-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-058-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-103-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-029-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-029-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-106-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-040-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-159-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-083-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-091-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-102-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-026-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-088-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-129-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-024-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-140-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-138-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-017-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-016-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-061-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-110-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-005-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-176-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-100-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-164-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-019-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-128-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-180-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-179-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-030-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-162-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-133-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-061-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-002-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-081-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-179-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-164-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-007-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-064-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-051-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-185-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-027-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-033-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-014-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-028-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-040-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-123-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-174-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-030-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-044-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-027-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-000-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-185-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-097-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-052-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-131-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-017-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-128-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-022-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-112-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-123-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-186-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-025-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-129-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-073-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-040-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-179-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-057-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-097-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-147-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-148-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-157-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-011-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-074-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-015-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-151-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-007-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-070-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-122-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-043-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-049-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-055-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-154-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-094-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-119-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-010-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-017-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-073-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-144-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-006-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-101-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-035-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-038-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-172-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-085-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-157-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-049-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-168-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-012-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-182-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-028-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-187-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-110-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-118-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-147-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-041-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-056-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-047-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-043-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-121-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-126-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-035-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-140-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-049-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-110-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-023-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-173-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-134-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-043-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-104-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-012-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-115-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-119-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-100-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-171-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-134-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-031-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-047-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-087-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-123-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-153-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-184-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-115-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-130-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-155-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-138-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-075-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-160-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-084-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-036-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-031-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-095-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-183-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-042-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-171-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-139-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-091-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-011-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-050-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-104-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-071-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-106-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-187-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-070-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-123-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-008-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-072-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-119-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-181-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-041-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-101-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-066-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-148-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-012-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-126-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-085-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-177-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-088-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-055-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-177-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-005-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-080-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-026-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-035-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-037-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-146-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-150-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-156-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-181-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-153-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-015-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-020-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-069-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-128-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-169-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-150-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-175-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-131-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-146-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-105-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-050-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-058-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-097-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-133-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-158-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-158-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-114-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-180-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-156-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-045-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-058-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-079-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-105-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-085-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-071-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-115-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-150-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-075-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-069-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-174-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-066-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-110-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-076-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-077-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-127-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-183-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-183-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-171-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-006-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-028-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-093-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-065-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-078-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-071-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-025-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-136-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-127-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-187-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-178-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-025-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-111-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-170-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-098-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-038-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-118-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-135-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-082-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-166-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-110-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-069-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-074-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-058-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-153-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-067-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-117-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-111-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-038-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-090-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-112-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-077-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-113-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-059-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-048-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-031-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-089-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-004-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-089-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-119-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-009-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-088-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-045-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-013-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-060-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-124-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-166-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-004-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-027-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-102-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-092-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-124-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-152-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-042-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-012-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-090-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-182-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-187-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-117-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-176-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-057-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-173-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-079-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-115-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-177-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-041-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-046-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-147-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-045-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-112-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-182-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-044-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-078-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-082-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-046-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-020-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-069-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-136-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-161-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-027-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-022-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-043-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-021-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-055-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-150-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-049-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-095-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-068-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-154-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-004-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-138-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-178-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-046-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-072-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-153-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-067-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-055-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-076-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-170-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-008-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-126-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-151-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-113-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-090-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-094-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-054-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-046-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-022-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-039-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-132-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-113-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-174-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-131-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-127-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-042-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-179-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-038-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-130-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-041-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-032-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-036-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-048-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-075-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-098-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-025-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-171-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-027-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-071-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-069-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-065-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-004-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-030-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-070-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-063-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-089-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-105-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-185-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-017-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-026-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-120-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-109-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-038-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-184-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-108-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-089-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-091-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-068-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-145-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-049-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-162-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-044-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-016-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-003-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-053-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-157-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-143-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-106-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-078-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-084-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-145-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-165-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-174-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-178-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-165-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-176-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-080-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-125-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-158-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-145-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-085-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-149-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-183-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-120-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-059-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-093-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-117-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-007-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-048-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-143-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-060-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-011-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-037-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-092-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-112-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-185-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-002-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-099-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-156-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-132-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-074-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-087-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-061-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-125-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-172-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-053-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-076-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-072-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-035-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-029-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-155-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-137-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-057-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-099-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-144-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-021-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-063-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-128-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-129-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-041-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-094-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-096-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-163-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-092-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-148-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-077-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-160-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-108-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-114-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-033-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-121-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-174-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-095-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-048-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-014-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-109-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-131-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-137-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-137-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-037-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-057-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-155-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-037-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-066-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-149-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-159-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-167-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-082-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-172-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-130-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-073-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-087-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-080-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-071-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-062-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-013-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-171-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-079-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-003-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-106-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-032-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-066-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-161-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-135-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-146-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-186-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-118-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-001-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-066-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-023-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-060-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-074-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-032-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-123-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-047-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-129-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-130-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-184-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-000-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-156-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-187-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-170-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-097-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-034-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-052-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-107-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-118-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-124-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-163-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-019-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-169-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-147-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-185-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-103-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-044-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-036-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-116-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-136-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-101-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-101-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-032-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-132-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-141-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-156-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-090-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-144-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-163-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-103-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-169-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-096-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-162-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-113-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-048-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-160-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-155-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-160-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-003-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-024-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-136-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-016-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-155-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-140-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-144-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-014-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-141-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-180-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-092-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-034-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-035-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-116-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-042-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-001-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-018-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-006-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-030-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-169-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-068-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-034-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-143-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-003-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-033-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-157-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-108-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-072-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-039-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-053-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-059-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-121-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-073-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-152-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-122-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-062-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-122-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-125-00005.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-129-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-083-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-075-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-107-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-080-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-086-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-085-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-098-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-127-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-181-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-104-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-037-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-137-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-000-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-007-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-161-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-166-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-008-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-175-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-020-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-139-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-021-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-152-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-025-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-042-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-141-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-023-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-058-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-122-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-022-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-076-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-095-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-137-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-001-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-086-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-149-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-031-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-054-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-062-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-151-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-105-00004.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-177-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-181-00002.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-100-00003.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-173-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-172-00000.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-098-00001.npy
+    - /weka/oe-training-default/ai2-llm/preprocessed/fastdclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/part-125-00001.npy
diff --git a/olmo/config.py b/olmo/config.py
index 94e5103d2..96c9ce029 100644
--- a/olmo/config.py
+++ b/olmo/config.py
@@ -19,6 +19,7 @@
 
 import numpy as np
 import torch
+import torch.nn.functional as F
 from omegaconf import DictConfig, ListConfig
 from omegaconf import OmegaConf as om
 from omegaconf.errors import OmegaConfBaseException
@@ -198,6 +199,11 @@ class BlockType(StrEnum):
     implementations of operations like attention to imitate the behavior of Llama.
     """
 
+    moe = "moe"
+    """
+    A block for OLMoE-style Mixture-of-Experts models.
+    """
+
 
 class InitFnType(StrEnum):
     mitchell = "mitchell"
@@ -457,6 +463,56 @@ class ModelConfig(BaseConfig):
     See :data:`TrainConfig.precision` instead.
     """
 
+    moe_num_experts: Optional[int] = 8
+    """
+    The number of experts to use in the MoE block.
+    """
+
+    moe_top_k: Optional[int] = 2
+    """
+    The number of experts to select for each token.
+    """
+
+    moe_mlp_impl: Optional[str] = "sparse"
+    """
+    Choose "grouped" for grouped GEMM installable via `pip install git+https://git@github.com/tgale96/grouped_gemm.git@66c7195e35e8c4f22fa6a014037ef511bfa397cb`.
+    """
+
+    moe_log_expert_assignment: Optional[bool] = True
+    """
+    Whether to log the expert assignment.
+    """
+
+    moe_shared_expert: Optional[bool] = False
+    """
+    Whether to have an always-used expert like in [DeepSeekMoE](https://arxiv.org/abs/2401.06066).
+    """
+
+    moe_lbl_in_fp32: Optional[bool] = False
+    """
+    Whether to perform load balancing in FP32.
+    """
+
+    moe_loss_weight: Optional[float] = 0.1
+    """
+    The weight to use for the MoE load balancing loss.
+    """
+
+    moe_zloss_weight: Optional[float] = None
+    """
+    Weight for MoE router z-loss where None means no router z-loss. 0.001 is a common value.
+    """
+
+    moe_dropless: Optional[bool] = True
+    """
+    Whether to use [dMoE](https://arxiv.org/abs/2211.15841).
+    """
+
+    moe_capacity_factor: Optional[float] = 1.25
+    """
+    The capacity factor to use in the MoE block. Only applies if not using dMoE.
+    """
+
     scale_emb_init: bool = False
     """
     If ``True``, embeddings are scaled up by ``sqrt(d_model)`` during initialization.
@@ -1283,3 +1339,41 @@ def update_legacy_settings(cls, config: D) -> D:
                 new_config.optimizer = OptimizerConfig.update_legacy_settings(new_config.optimizer)
 
         return new_config
+
+
+def config_to_moe_args(config: ModelConfig) -> Dict[str, Any]:
+    from megablocks.layers.arguments import Arguments as MoEArgs
+
+    from .model import Activation
+
+    hidden_size = (
+        config.mlp_hidden_size if config.mlp_hidden_size is not None else config.mlp_ratio * config.d_model
+    )
+    act = Activation.build(config)
+    kwargs = {
+        "activation_fn": F.silu if "swiglu" in config.activation_type.lower() else Activation.build(config),
+        "mlp_type": "glu" if "glu" in config.activation_type.lower() else "mlp",
+        "mlp_impl": config.moe_mlp_impl,
+        "hidden_size": config.d_model,
+        "ffn_hidden_size": int(act.output_multiplier * hidden_size),
+        "moe_num_experts": config.moe_num_experts,
+        "num_layers": config.n_layers,
+        # Handled by FSDP (https://github.com/databricks/megablocks/issues/57#issuecomment-1854594483)
+        "moe_weight_parallelism": False,
+        "moe_expert_model_parallelism": False,
+        "moe_top_k": config.moe_top_k,
+        "moe_capacity_factor": config.moe_capacity_factor,
+        "moe_loss_weight": config.moe_loss_weight,
+        "device": config.init_device,
+        # Handled by FSDP
+        "bf16": False,
+        "fp16": False,
+        "bias": config.include_bias,
+        "return_bias": False,
+        "shared_expert": config.moe_shared_expert,
+        "moe_lbl_in_fp32": config.moe_lbl_in_fp32,
+    }
+    if config.moe_zloss_weight:
+        kwargs["moe_zloss_weight"] = config.moe_zloss_weight
+
+    return MoEArgs(**kwargs)
diff --git a/olmo/initialization.py b/olmo/initialization.py
index 2a06581a5..f203497f6 100644
--- a/olmo/initialization.py
+++ b/olmo/initialization.py
@@ -13,9 +13,15 @@ def init_normal(
     # weights
     if init_cutoff_factor is not None:
         cutoff_value = init_cutoff_factor * std
-        nn.init.trunc_normal_(module.weight, mean=0.0, std=std, a=-cutoff_value, b=cutoff_value)
+        if hasattr(module, "weight"):
+            nn.init.trunc_normal_(module.weight, mean=0.0, std=std, a=-cutoff_value, b=cutoff_value)
+        else:
+            nn.init.trunc_normal_(module, mean=0.0, std=std, a=-cutoff_value, b=cutoff_value)
     else:
-        nn.init.normal_(module.weight, mean=0.0, std=std)
+        if hasattr(module, "weight"):
+            nn.init.normal_(module.weight, mean=0.0, std=std)
+        else:
+            nn.init.normal_(module, mean=0.0, std=std)
 
     # biases
     if isinstance(module, nn.Linear) and module.bias is not None:
diff --git a/olmo/model.py b/olmo/model.py
index ab7ed9f0d..731aca737 100644
--- a/olmo/model.py
+++ b/olmo/model.py
@@ -448,14 +448,15 @@ def __init__(self, layer_id: int, config: ModelConfig, cache: BufferCache):
             config.d_model, config.d_model, bias=config.include_bias, device=config.init_device
         )
 
-        # Feed-forward output projection.
-        self.ff_out = nn.Linear(
-            int(self.act.output_multiplier * self.hidden_size),
-            config.d_model,
-            bias=config.include_bias,
-            device=config.init_device,
-        )
-        self.ff_out._is_residual = True  # type: ignore
+        if self.config.block_type != BlockType.moe:
+            # Feed-forward output projection.
+            self.ff_out = nn.Linear(
+                int(self.act.output_multiplier * self.hidden_size),
+                config.d_model,
+                bias=config.include_bias,
+                device=config.init_device,
+            )
+            self.ff_out._is_residual = True  # type: ignore
 
         # Rotary embeddings.
         if self.config.rope:
@@ -664,10 +665,164 @@ def build(cls, layer_id: int, config: ModelConfig, cache: BufferCache) -> OLMoBl
             return OLMoSequentialBlock(layer_id, config, cache)
         elif config.block_type == BlockType.llama:
             return OLMoLlamaBlock(layer_id, config, cache)
+        elif config.block_type == BlockType.moe:
+            return OLMoEBlock(layer_id, config, cache)
         else:
             raise NotImplementedError(f"Unknown block type: '{config.block_type}'")
 
 
+class OLMoEBlock(OLMoBlock):
+    """
+    This is a transformer MoE block where the output is computed as ``MoE(LN(x + Attention(LN(x))))``
+    (plus another skip connection).
+    """
+
+    def __init__(self, layer_id: int, config: ModelConfig, cache: BufferCache):
+        try:
+            from megablocks.layers.dmoe import dMoE
+            from megablocks.layers.moe import MoE
+        except ImportError:
+            raise ImportError(
+                "To train MoEs, run `pip install git+https://github.com/Muennighoff/megablocks.git@olmoe`"
+            )
+        from .config import config_to_moe_args
+
+        super().__init__(layer_id, config, cache)
+
+        self.moe_args = config_to_moe_args(config)
+        self.ffn = dMoE(self.moe_args) if self.config.moe_dropless else MoE(self.moe_args)
+
+        self.attn_norm = LayerNorm.build(config)
+        self.ff_norm = LayerNorm.build(config)
+
+        # Attention input projection. Projects x -> (q, k, v)
+        head_dim = config.d_model // config.n_heads
+        self.fused_dims = (
+            config.d_model,
+            config.effective_n_kv_heads * head_dim,
+            config.effective_n_kv_heads * head_dim,
+        )
+        self.att_proj = nn.Linear(
+            config.d_model, sum(self.fused_dims), bias=config.include_bias, device=config.init_device
+        )
+
+    def reset_parameters(self):
+        if self.k_norm is not None:
+            self.k_norm.reset_parameters()
+        if self.q_norm is not None:
+            self.q_norm.reset_parameters()
+
+        if self.config.init_fn == InitFnType.normal:
+            attn_out_std = ff_out_std = in_std = self.config.init_std
+            cutoff_factor = self.config.init_cutoff_factor
+        elif self.config.init_fn == InitFnType.mitchell:
+            in_std = 1 / math.sqrt(self.config.d_model)
+            attn_out_std = 1 / (math.sqrt(2 * self.config.d_model * (self.layer_id + 1)))
+            ff_out_std = 1 / (math.sqrt(2 * self.ff_out.in_features * (self.layer_id + 1)))
+            cutoff_factor = self.config.init_cutoff_factor or 3.0
+        elif self.config.init_fn == InitFnType.full_megatron:
+            in_std = self.config.init_std
+            attn_out_std = ff_out_std = self.config.init_std / math.sqrt(2.0 * self.config.n_layers)
+            cutoff_factor = self.config.init_cutoff_factor or 3.0
+        else:
+            raise NotImplementedError(self.config.init_fn)
+
+        init_normal(self.att_proj, std=in_std, init_cutoff_factor=cutoff_factor)
+        init_normal(self.attn_out, std=attn_out_std, init_cutoff_factor=cutoff_factor)
+        self.attn_norm.reset_parameters()
+        self.ff_norm.reset_parameters()
+        init_normal(self.ffn.experts.mlp.w1, std=in_std, init_cutoff_factor=cutoff_factor)
+        init_normal(self.ffn.experts.mlp.w2, std=ff_out_std, init_cutoff_factor=cutoff_factor)
+        if hasattr(self.ffn.experts.mlp, "v1"):
+            init_normal(self.ffn.experts.mlp.v1, std=in_std, init_cutoff_factor=cutoff_factor)
+        if self.ffn.experts.bias is not None:
+            torch.nn.init.zeros_(self.ffn.experts.bias)
+        init_normal(self.ffn.router.layer, std=in_std, init_cutoff_factor=cutoff_factor)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        attention_bias: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        use_cache: bool = False,
+        max_doc_len: Optional[int] = None,
+        cu_doc_lens: Optional[torch.Tensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        # Get query, key, value projections.
+        # shape:
+        #  - for regular attn q, k, v: (batch_size, seq_len, d_model)
+        #  - for multi-query attn q: (batch_size, seq_len, d_model)
+        #                      k, v: (batch_size, seq_len, d_model // n_heads)
+        #  - for group query attn q: (batch_size, seq_len, d_model)
+        #                      k, v: (batch_size, seq_len, d_model // n_kv_heads)
+        if not self.config.norm_after:
+            if self._activation_checkpoint_fn is not None:
+                qkv = self.att_proj(self._activation_checkpoint_fn(self.attn_norm, x))
+            else:
+                qkv = self.att_proj(self.attn_norm(x))
+        else:
+            qkv = self.att_proj(x)
+
+        if self.config.clip_qkv is not None:
+            qkv.clamp_(min=-self.config.clip_qkv, max=self.config.clip_qkv)
+
+        q, k, v = qkv.split(self.fused_dims, dim=-1)
+
+        # Get attention scores.
+        if self._activation_checkpoint_fn is not None:
+            att, cache = self._activation_checkpoint_fn(  # type: ignore
+                self.attention,
+                q,
+                k,
+                v,
+                attention_bias,
+                layer_past=layer_past,
+                use_cache=use_cache,
+                max_doc_len=max_doc_len,
+                cu_doc_lens=cu_doc_lens,
+            )
+        else:
+            att, cache = self.attention(
+                q,
+                k,
+                v,
+                attention_bias,
+                layer_past=layer_past,
+                use_cache=use_cache,
+                max_doc_len=max_doc_len,
+                cu_doc_lens=cu_doc_lens,
+            )
+
+        if self.config.norm_after:
+            if self._activation_checkpoint_fn is not None:
+                att = self._activation_checkpoint_fn(self.attn_norm, att)
+            else:
+                att = self.attn_norm(att)
+
+        # Add attention scores.
+        # shape: (B, T, C)
+        x = x + self.dropout(att)
+
+        # Add feed-forward projection.
+        # shape: (batch_size, seq_len, d_model)
+        og_x = x
+
+        if self.config.norm_after:
+            x = self.ffn(x)
+            if self._activation_checkpoint_fn is not None:
+                x = self._activation_checkpoint_fn(self.ff_norm, x)  # type: ignore
+            else:
+                x = self.ff_norm(x)
+            return og_x + self.dropout(x), cache
+        else:
+            if self._activation_checkpoint_fn is not None:
+                x = self._activation_checkpoint_fn(self.ff_norm, x)  # type: ignore
+            else:
+                x = self.ff_norm(x)
+            # Activation checkpointing for the MoE FFN is not supported
+            return og_x + self.dropout(self.ffn(x)), cache
+
+
 class OLMoSequentialBlock(OLMoBlock):
     """
     This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
@@ -1552,7 +1707,7 @@ def fsdp_wrap_fn(module, recurse: bool = True, nonwrapped_numel: int = 0):
         else:
             raise NotImplementedError(wrap_strategy)
 
-    def num_params(self, include_embedding: bool = True) -> int:
+    def num_params(self, include_embedding: bool = True, include_inactive_params: bool = True) -> int:
         """
         Get the total number of parameters.
         """
@@ -1562,6 +1717,15 @@ def num_params(self, include_embedding: bool = True) -> int:
                 lambda np: ".wte." not in np[0] and ".wpe." not in np[0],
                 params,
             )
+        if not include_inactive_params:
+            # Need to reduce blocks to the number of experts that are selected
+            # If not dropless 'transformer.blocks.0.ffn.experts.mlp.w1' has shape (total_experts, in_dim, out_dim)
+            # change to 'transformer.blocks.0.ffn.experts.mlp.w1' with shape (selected_experts, in_dim, out_dim)
+            # If dropless, the total_experts & out_dim are combined into one dimension
+            idx = self.config.moe_top_k
+            if self.config.moe_dropless:
+                idx *= self.transformer.blocks[1].moe_args.ffn_hidden_size
+            params = [(np[0], np[1][:idx]) if "experts.mlp" in np[0] else np for np in params]  # type: ignore
         return sum(p.numel() for _, p in params)
 
     @property
diff --git a/olmo/optim.py b/olmo/optim.py
index 5460ccee1..e399c2e9c 100644
--- a/olmo/optim.py
+++ b/olmo/optim.py
@@ -15,6 +15,13 @@
 from .config import OptimizerType, SchedulerConfig, SchedulerType, TrainConfig
 from .torch_util import get_default_device, is_distributed
 
+try:
+    from megablocks.layers.mlp import MLP, SparseMLP
+
+    megablocks_available = True
+except ImportError:
+    megablocks_available = False
+
 __all__ = [
     "Optimizer",
     "LionW",
@@ -858,6 +865,8 @@ def get_param_groups(cfg: TrainConfig, model: nn.Module) -> List[Dict[str, Any]]
                     decay.add(fpn)
                 else:
                     no_decay.add(fpn)
+            elif megablocks_available and pn.endswith(("w1", "w2", "v1")) and isinstance(m, (MLP, SparseMLP)):
+                decay.add(fpn)
 
     # Validate that we've considered every parameter
     inter_params = decay & no_decay
diff --git a/olmo/train.py b/olmo/train.py
index 341055003..ec458a24c 100644
--- a/olmo/train.py
+++ b/olmo/train.py
@@ -15,7 +15,18 @@
 from itertools import islice
 from pathlib import Path
 from pstats import SortKey
-from typing import Any, Callable, Deque, Dict, List, Optional, TextIO, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Deque,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    TextIO,
+    Tuple,
+    Union,
+)
 
 import numpy as np
 import torch
@@ -32,6 +43,7 @@
 from .aliases import PathOrStr
 from .checkpoint import Checkpointer, FullCheckpointer, build_sharded_checkpointer
 from .config import (
+    BlockType,
     CheckpointType,
     DDPGradSyncMode,
     DistributedStrategy,
@@ -58,6 +70,15 @@
 )
 from .util import upload
 
+try:
+    from megablocks.layers.moe import (
+        batched_load_balancing_loss,
+        clear_load_balancing_loss,
+        get_load_balancing_loss,
+    )
+except ImportError:
+    pass
+
 __all__ = ["SpeedMonitor", "LRMonitor", "Trainer"]
 
 log = logging.getLogger(__name__)
@@ -240,6 +261,11 @@ def __post_init__(self):
             else:
                 raise NameError("`fused_loss_fn` is not defined. Please ensure that `flash_attn` is installed.")
 
+        if self.model.config.block_type == BlockType.moe:
+            from .config import config_to_moe_args
+
+            self.moe_args = config_to_moe_args(self.cfg.model)
+
     @property
     def dataset(self) -> IterableDataset:
         assert isinstance(self.train_loader.dataset, IterableDataset)
@@ -767,7 +793,15 @@ def train_micro_batch(
 
         return loss, ce_loss, z_loss
 
-    def train_batch(self, batch: Dict[str, Any]) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+    def train_batch(
+        self, batch: Dict[str, Any]
+    ) -> Tuple[
+        torch.Tensor,
+        Optional[torch.Tensor],
+        Optional[torch.Tensor],
+        Optional[torch.Tensor],
+        Optional[Iterable[Any]],
+    ]:
         # Split into micro-batches.
         micro_batches = self.split_batch(batch)
         batch_size_in_tokens = batch["input_ids"].numel()
@@ -777,6 +811,20 @@ def train_batch(self, batch: Dict[str, Any]) -> Tuple[torch.Tensor, Optional[tor
 
         ce_batch_loss = torch.tensor(0.0, device=self.device)
         z_batch_loss = None if not self.cfg.softmax_auxiliary_loss else torch.tensor(0.0, device=self.device)
+        lb_batch_loss = (
+            None if self.model.config.block_type != BlockType.moe else torch.tensor(0.0, device=self.device)
+        )
+        moe_z_batch_loss = (
+            None if not self.model.config.moe_zloss_weight else torch.tensor(0.0, device=self.device)
+        )
+        expert_assignments = (
+            None
+            if (
+                (self.model.config.block_type != BlockType.moe)
+                or (self.model.config.moe_log_expert_assignment is False)
+            )
+            else torch.zeros((self.model.config.n_layers, self.model.config.moe_num_experts), device=self.device)
+        )
         num_micro_batches = len(micro_batches)
 
         for micro_batch_idx, micro_batch in enumerate(micro_batches):
@@ -807,6 +855,27 @@ def train_batch(self, batch: Dict[str, Any]) -> Tuple[torch.Tensor, Optional[tor
                         assert z_batch_loss is not None
                         z_batch_loss += z_loss.detach()
 
+                if self.model.config.block_type == BlockType.moe:
+                    if self.model.config.moe_zloss_weight:
+                        lb_loss, moe_z_loss = batched_load_balancing_loss(self.moe_args)
+                        lb_loss = lb_loss / len(micro_batches)
+                        moe_z_loss = moe_z_loss / len(micro_batches)
+                    elif self.model.config.moe_loss_weight:
+                        lb_loss = batched_load_balancing_loss(self.moe_args) / len(micro_batches)
+                    if self.model.config.moe_log_expert_assignment:
+                        if self.model.config.moe_zloss_weight:
+                            tokens_per_expert, _, _ = zip(*get_load_balancing_loss())
+                        else:
+                            tokens_per_expert, _ = zip(*get_load_balancing_loss())
+                        expert_assignments += torch.stack(tokens_per_expert, dim=0)
+                    clear_load_balancing_loss()
+                    if self.model.config.moe_loss_weight:
+                        loss += lb_loss
+                        lb_batch_loss += lb_loss.detach()
+                    if self.model.config.moe_zloss_weight:
+                        loss += moe_z_loss
+                        moe_z_batch_loss += moe_z_loss.detach()
+
                 # Run backward pass.
                 loss.backward()
 
@@ -814,7 +883,7 @@ def train_batch(self, batch: Dict[str, Any]) -> Tuple[torch.Tensor, Optional[tor
             for hook in output_hooks:
                 hook.remove()
 
-        return ce_batch_loss, z_batch_loss
+        return ce_batch_loss, z_batch_loss, lb_batch_loss, moe_z_batch_loss, expert_assignments
 
     def train_step(self, batch: Dict[str, Any], reduce_global_loss: bool = True) -> Dict[str, float]:
         metrics: Dict[str, float] = {}
@@ -835,7 +904,7 @@ def train_step(self, batch: Dict[str, Any], reduce_global_loss: bool = True) ->
         batch = move_to_device(batch, self.device)
 
         # Run forward-backward pass.
-        ce_batch_loss, z_batch_loss = self.train_batch(batch)
+        ce_batch_loss, z_batch_loss, lb_batch_loss, moe_z_batch_loss, expert_assignments = self.train_batch(batch)
 
         # Collect loss, potentially reducing over all ranks.
         if reduce_global_loss:
@@ -844,6 +913,12 @@ def train_step(self, batch: Dict[str, Any], reduce_global_loss: bool = True) ->
             if z_batch_loss is not None:
                 dist.reduce(z_batch_loss, 0)
                 z_batch_loss.div_(get_world_size())
+            if lb_batch_loss is not None:
+                dist.reduce(lb_batch_loss, 0)
+                lb_batch_loss.div_(get_world_size())
+            if moe_z_batch_loss is not None:
+                dist.reduce(moe_z_batch_loss, 0)
+                moe_z_batch_loss.div_(get_world_size())
 
         # Clip gradient norms and collect param/gradient/optim metrics.
         should_log_optim_metrics_this_step = self.should_log_optim_metrics_this_step()
@@ -887,6 +962,21 @@ def train_step(self, batch: Dict[str, Any], reduce_global_loss: bool = True) ->
         metrics["train/Perplexity"] = math.exp(self.cur_train_loss)
         if z_batch_loss is not None:
             metrics["train/ZLoss"] = z_batch_loss.item()
+        if lb_batch_loss is not None:
+            metrics["train/LoadBalancingLoss"] = lb_batch_loss.item()
+            # Log assignment metrics.
+            if expert_assignments is not None:
+                for layer_idx, expert_assignments_layer in enumerate(expert_assignments):
+                    total_tokens = expert_assignments_layer.sum().item()
+                    for expert_idx, expert_assignment in enumerate(expert_assignments_layer):
+                        metrics[f"train/TokensPercentage/layer{layer_idx}/expert{expert_idx}"] = (
+                            expert_assignment.item() / total_tokens
+                        ) * 100
+                        metrics[
+                            f"train/TokensTotal/layer{layer_idx}/expert{expert_idx}"
+                        ] = expert_assignment.item()
+        if moe_z_batch_loss is not None:
+            metrics["train/MoEZLoss"] = moe_z_batch_loss.item()
 
         # Maybe collect post-step optimizer-specific metrics.
         if should_log_optim_metrics_this_step:
diff --git a/scripts/train.py b/scripts/train.py
index ff7bb31b8..eeb6a82e4 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -17,6 +17,7 @@
 from torch.nn.parallel import DistributedDataParallel as DDP
 
 from olmo.config import (
+    ActivationCheckpointingStrategy,
     CheckpointType,
     DDPGradSyncMode,
     DistributedStrategy,
@@ -127,9 +128,20 @@ def main(cfg: TrainConfig) -> None:
 
     # Initialize the model.
     log.info("Building model...")
+    if (
+        cfg.model.block_type == "moe"
+        and cfg.activation_checkpointing
+        and cfg.activation_checkpointing != ActivationCheckpointingStrategy.fine_grained
+    ):
+        raise OLMoConfigurationError(
+            "Only no or fine-grained activation checkpointing is supported for MoE models."
+        )
+
     olmo_model = OLMo(cfg.model)
     log.info(f"Total number of parameters: {olmo_model.num_params():,d}")
     log.info(f"Number of non-embedding parameters: {olmo_model.num_params(include_embedding=False):,d}")
+    if olmo_model.config.block_type == "moe":
+        log.info(f"Number of active parameters: {olmo_model.num_params(include_inactive_params=False):,d}")
     log.info(f"Peak GPU Memory (MB) before {cfg.distributed_strategy}: {int(peak_gpu_memory() or 0)}")
 
     olmo_model.set_activation_checkpointing(cfg.activation_checkpointing)