Merge pull request #255 from Modalities/megatron_dataset_removal

refactor: removed legacy megatron dataset code
Modalities · Sep 17, 2024 · ebb75bc · ebb75bc
2 parents 8158de7 + 52172be
commit ebb75bc
Show file tree

Hide file tree

Showing 3 changed files with 0 additions and 169 deletions.
diff --git a/benchmarks/dataloader/README.md b/benchmarks/dataloader/README.md
diff --git a/benchmarks/dataloader/launch_benchmark.sh b/benchmarks/dataloader/launch_benchmark.sh
diff --git a/src/modalities/config/config.py b/src/modalities/config/config.py
@@ -285,11 +285,6 @@ class PackedMemMapDatasetMegatronConfig(BaseModel):
     sample_key: str
 
 
-class MMapIndexedDatasetConfig(BaseModel):
-    path: Path
-    skip_warmup: bool
-
-
 class BatchSamplerConfig(BaseModel):
     sampler: PydanticSamplerIFType
     batch_size: Annotated[int, Field(strict=True, gt=0)]