Remove shuffling from tfreader as it is already correctly done during…

… reconfigure.
argonne-lcf · Jul 30, 2024 · 1510dfd · 1510dfd
1 parent 82c0796
commit 1510dfd
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 4 deletions.
diff --git a/dlio_benchmark/data_loader/tf_data_loader.py b/dlio_benchmark/data_loader/tf_data_loader.py
@@ -95,7 +95,7 @@ def read(self):
             self._dataset = ReaderFactory.get_reader(type=self.format_type,
                                           dataset_type=self.dataset_type,
                                           thread_index=-1,
-                                          epoch_number=0).next()
+                                          epoch_number=self.epoch_number).next()
 
     @dlp.log
     def next(self):

diff --git a/dlio_benchmark/reader/tf_reader.py b/dlio_benchmark/reader/tf_reader.py
@@ -81,12 +81,13 @@ def _parse_image(self, serialized):
 
     @dlp.log
     def next(self):
-        logging.debug(f"{utcnow()} Reading {len(self._file_list)} files thread {self.thread_index} rank {self._args.my_rank}")
-        filenames = tf.data.Dataset.list_files(self._file_list, shuffle=True)
+        logging.debug(f"{utcnow()} Reading {self._file_list} files thread {self.thread_index} rank {self._args.my_rank}")
+        filenames = tf.data.Dataset.list_files(self._file_list, shuffle=False)
         # sharding in the file list if we have enought files. 
         if (len(self._file_list) >= self._args.comm_size):
             filenames = filenames.shard(num_shards=self._args.comm_size, index=self._args.my_rank)
-
+            logging.debug(f"{utcnow()} shard {filenames} files index {self._args.my_rank} number {self._args.comm_size}")
+
         self._dataset = tf.data.TFRecordDataset(filenames=filenames, buffer_size=self._args.transfer_size,
                                                 num_parallel_reads=self._args.read_threads)