Don't reshuffle eval data each "epoch"

olmo/data/__init__.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from torch.utils.data import DataLoader, DistributedSampler
 
@@ -15,14 +15,15 @@
 
 def build_memmap_dataset(train_config: TrainConfig, data_config: DataConfig) -> MemMapDataset:
     paths: List[str]
-    metadata: Optional[List[Dict[str, Any]]] = None
+    metadata: List[Dict[str, Any]] = []
     if data_config.paths:
         if data_config.datasets:
             raise OlmoConfigurationError("DataConfig.paths is mutually exclusive with DataConfig.datasets")
         paths = data_config.paths
+        for path in paths:
+            metadata.append({"path": str(path)})
     elif data_config.datasets:
         paths = []
-        metadata = []
         for label in sorted(data_config.datasets.keys()):
             label_paths = data_config.datasets[label]
             paths.extend(label_paths)

olmo/eval/__init__.py

@@ -97,7 +97,7 @@ def make_metric():
             label=eval_config.label,
             type=eval_config.type,
             eval_loader=eval_loader,
-            eval_batches=cycle_through_epochs(eval_loader),
+            eval_batches=cycle_through_epochs(eval_loader, update_epoch_seed=False),
             eval_metric=eval_metric,
             subset_num_batches=eval_config.subset_num_batches,
         )

olmo/util.py

@@ -333,12 +333,14 @@ def peak_gpu_memory(reset: bool = False) -> Optional[float]:
     return peak_mb
 
 
-def cycle_through_epochs(dataloader: DataLoader) -> Generator[Dict[str, Any], None, None]:
+def cycle_through_epochs(
+    dataloader: DataLoader, update_epoch_seed: bool = True
+) -> Generator[Dict[str, Any], None, None]:
     while True:
         for batch in dataloader:
             yield batch
 
-        if isinstance(dataloader.sampler, DistributedSampler):
+        if update_epoch_seed and isinstance(dataloader.sampler, DistributedSampler):
             epoch = dataloader.sampler.epoch + 1
             dataloader.sampler.set_epoch(epoch)