Merge branch 'main' into metric

dlio_benchmark/checkpointing/pytorch_checkpointing.py

@@ -18,7 +18,7 @@
 import torch
 
 from dlio_benchmark.checkpointing.base_checkpointing import BaseCheckpointing
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 from dlio_benchmark.common.constants import MODULE_CHECKPOINT
 from dlio_benchmark.common.enumerations import CheckpointLocationType

dlio_benchmark/checkpointing/tf_checkpointing.py

@@ -17,7 +17,7 @@
 import os
 
 from dlio_benchmark.checkpointing.base_checkpointing import BaseCheckpointing
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 import tensorflow as tf
 
 from dlio_benchmark.common.constants import MODULE_CHECKPOINT

dlio_benchmark/configs/workload/cosmoflow_a100.yaml

@@ -18,7 +18,13 @@ reader:
  data_loader: tensorflow
  read_threads: 4
  batch_size: 1
+ file_shuffle: seed
+ sample_shuffle: seed
+ shuffle_size: 2
 
 train: 
   epochs: 5
   computation_time: 0.00551
+
+metric:
+ au: 0.70

dlio_benchmark/configs/workload/cosmoflow_h100.yaml

@@ -18,7 +18,13 @@ reader:
  data_loader: tensorflow
  read_threads: 4
  batch_size: 1
-
+ file_shuffle: seed
+ sample_shuffle: seed
+ shuffle_size: 2
+
 train: 
   epochs: 5
   computation_time: 0.00350
+
+metric:
+  au: 0.70

dlio_benchmark/configs/workload/resnet50_a100.yaml

@@ -23,4 +23,7 @@ reader:
  read_threads: 8
  computation_threads: 8
  batch_size: 400
- dont_use_mmap: True
+ dont_use_mmap: True
+
+metric:
+ au: 0.90

dlio_benchmark/configs/workload/resnet50_h100.yaml

@@ -23,4 +23,7 @@ reader:
  read_threads: 8
  computation_threads: 8
  batch_size: 400
- dont_use_mmap: True
+
+
+metric:
+ au: 0.90

dlio_benchmark/configs/workload/unet3d_a100.yaml

@@ -32,3 +32,6 @@ checkpoint:
   checkpoint_after_epoch: 5
   epochs_between_checkpoints: 2
   model_size: 499153191
+
+metric:
+  au: 0.90

dlio_benchmark/configs/workload/unet3d_h100.yaml

@@ -32,3 +32,6 @@ checkpoint:
   checkpoint_after_epoch: 5
   epochs_between_checkpoints: 2
   model_size: 499153191
+
+metric:
+  au: 0.90

dlio_benchmark/data_generator/hdf5_generator.py

@@ -21,7 +21,7 @@
 from dlio_benchmark.common.enumerations import Compression
 from dlio_benchmark.data_generator.data_generator import DataGenerator
 from dlio_benchmark.utils.utility import progress
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from shutil import copyfile
 
 from dlio_benchmark.common.constants import MODULE_DATA_GENERATOR

dlio_benchmark/data_generator/indexed_binary_generator.py

@@ -22,7 +22,7 @@
 import numpy as np
 
 from dlio_benchmark.utils.utility import progress, utcnow, DLIOMPI
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from shutil import copyfile
 from dlio_benchmark.common.constants import MODULE_DATA_GENERATOR
 import struct

dlio_benchmark/data_generator/jpeg_generator.py

@@ -22,7 +22,7 @@
 import numpy as np
 
 from dlio_benchmark.utils.utility import progress, utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from shutil import copyfile
 import PIL.Image as im
 from dlio_benchmark.common.constants import MODULE_DATA_GENERATOR

dlio_benchmark/data_generator/npy_generator.py

@@ -22,7 +22,7 @@
 import numpy as np
 
 from dlio_benchmark.utils.utility import progress, utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from shutil import copyfile
 from dlio_benchmark.common.constants import MODULE_DATA_GENERATOR
 

dlio_benchmark/data_generator/npz_generator.py

@@ -22,7 +22,7 @@
 import numpy as np
 
 from dlio_benchmark.utils.utility import progress, utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from shutil import copyfile
 from dlio_benchmark.common.constants import MODULE_DATA_GENERATOR
 

dlio_benchmark/data_generator/png_generator.py

@@ -22,7 +22,7 @@
 import numpy as np
 
 from dlio_benchmark.utils.utility import progress, utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from shutil import copyfile
 import PIL.Image as im
 from dlio_benchmark.common.constants import MODULE_DATA_GENERATOR

dlio_benchmark/data_generator/synthetic_generator.py

@@ -22,7 +22,7 @@
 import numpy as np
 
 from dlio_benchmark.utils.utility import progress, utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from shutil import copyfile
 import PIL.Image as im
 from dlio_benchmark.common.constants import MODULE_DATA_GENERATOR

dlio_benchmark/data_generator/tf_generator.py

@@ -20,7 +20,7 @@
 from dlio_benchmark.data_generator.data_generator import DataGenerator
 import numpy as np
 import tensorflow as tf
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 from dlio_benchmark.utils.utility import progress, utcnow
 from shutil import copyfile

dlio_benchmark/data_loader/dali_data_loader.py

@@ -27,7 +27,7 @@
 from dlio_benchmark.data_loader.base_data_loader import BaseDataLoader
 from dlio_benchmark.reader.reader_factory import ReaderFactory
 from dlio_benchmark.utils.utility import utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 import os
 
 dlp = Profile(MODULE_DATA_LOADER)

dlio_benchmark/data_loader/synthetic_data_loader.py

@@ -27,7 +27,7 @@
 from dlio_benchmark.data_loader.base_data_loader import BaseDataLoader
 from dlio_benchmark.reader.reader_factory import ReaderFactory
 from dlio_benchmark.utils.utility import utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 import os
 
 dlp = Profile(MODULE_DATA_LOADER)

dlio_benchmark/data_loader/tf_data_loader.py

@@ -25,7 +25,7 @@
 from dlio_benchmark.data_loader.base_data_loader import BaseDataLoader
 from dlio_benchmark.reader.reader_factory import ReaderFactory
 from dlio_benchmark.utils.utility import utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 import numpy as np
 

dlio_benchmark/data_loader/torch_data_loader.py

@@ -29,7 +29,7 @@
 from dlio_benchmark.reader.reader_factory import ReaderFactory
 from dlio_benchmark.utils.utility import utcnow, DLIOMPI
 from dlio_benchmark.utils.config import ConfigArguments
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_DATA_LOADER)
 

dlio_benchmark/framework/tf_framework.py

@@ -21,7 +21,7 @@
 from dlio_benchmark.common.constants import MODULE_AI_FRAMEWORK
 from dlio_benchmark.data_loader.data_loader_factory import DataLoaderFactory
 from dlio_benchmark.utils.utility import utcnow, DLIOMPI
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from dlio_benchmark.common.error_code import ErrorCodes
 from dlio_benchmark.framework.framework import Framework
 from dlio_benchmark.reader.reader_factory import ReaderFactory

dlio_benchmark/framework/torch_framework.py

@@ -25,7 +25,7 @@
 import functools
 import logging
 from dlio_benchmark.utils.utility import utcnow, DLIOMPI
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 from time import sleep, time
 

dlio_benchmark/main.py

@@ -16,7 +16,7 @@
 """
 import os
 import math
-import hydra
+
 import logging
 from time import time, sleep
 import json
@@ -46,10 +46,13 @@
 from dlio_benchmark.framework.framework_factory import FrameworkFactory
 from dlio_benchmark.data_generator.generator_factory import GeneratorFactory
 from dlio_benchmark.storage.storage_factory import StorageFactory
-from dlio_profiler.logger import dlio_logger as PerfTrace, fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile, PerfTrace
 
 dlp = Profile(MODULE_DLIO_BENCHMARK)
-
+from mpi4py import MPI
+# To make sure the output folder is the same in all the nodes. We have to do this. 
+MPI.COMM_WORLD.Barrier()
+import hydra
 
 class DLIOBenchmark(object):
     """

dlio_benchmark/reader/csv_reader.py

@@ -17,7 +17,7 @@
 import pandas as pd
 
 from dlio_benchmark.common.constants import MODULE_DATA_READER
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from dlio_benchmark.reader.reader_handler import FormatReader
 
 dlp = Profile(MODULE_DATA_READER)

dlio_benchmark/reader/hdf5_reader.py

@@ -19,7 +19,7 @@
 import h5py
 
 from dlio_benchmark.common.constants import MODULE_DATA_READER
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from dlio_benchmark.reader.reader_handler import FormatReader
 
 dlp = Profile(MODULE_DATA_READER)

dlio_benchmark/reader/image_reader.py

@@ -22,7 +22,7 @@
 from dlio_benchmark.common.constants import MODULE_DATA_READER
 from dlio_benchmark.reader.reader_handler import FormatReader
 from dlio_benchmark.utils.utility import utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_DATA_READER)
 

dlio_benchmark/reader/indexed_binary_mmap_reader.py

@@ -22,7 +22,7 @@
 from dlio_benchmark.common.constants import MODULE_DATA_READER
 from dlio_benchmark.common.enumerations import DataLoaderSampler
 from dlio_benchmark.reader.reader_handler import FormatReader
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_DATA_READER)
 

dlio_benchmark/reader/indexed_binary_reader.py

@@ -22,7 +22,7 @@
 from dlio_benchmark.common.constants import MODULE_DATA_READER
 from dlio_benchmark.common.enumerations import DataLoaderSampler
 from dlio_benchmark.reader.reader_handler import FormatReader
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_DATA_READER)
 

dlio_benchmark/reader/npy_reader.py

@@ -18,7 +18,7 @@
 
 from dlio_benchmark.common.constants import MODULE_DATA_READER
 from dlio_benchmark.reader.reader_handler import FormatReader
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_DATA_READER)
 

dlio_benchmark/reader/npz_reader.py

@@ -18,7 +18,7 @@
 
 from dlio_benchmark.common.constants import MODULE_DATA_READER
 from dlio_benchmark.reader.reader_handler import FormatReader
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_DATA_READER)
 

dlio_benchmark/reader/reader_handler.py

@@ -21,7 +21,7 @@
 from dlio_benchmark.framework.framework_factory import FrameworkFactory
 from dlio_benchmark.storage.storage_factory import StorageFactory
 from dlio_benchmark.utils.utility import utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from dlio_benchmark.utils.config import ConfigArguments
 import numpy as np
 import os

dlio_benchmark/reader/synthetic_reader.py

@@ -18,7 +18,7 @@
 
 from dlio_benchmark.common.constants import MODULE_DATA_READER
 from dlio_benchmark.reader.reader_handler import FormatReader
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_DATA_READER)
 

dlio_benchmark/reader/tf_reader.py

@@ -20,7 +20,7 @@
 
 from dlio_benchmark.common.constants import MODULE_DATA_READER
 from dlio_benchmark.utils.utility import utcnow
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 from dlio_benchmark.common.enumerations import DatasetType, Shuffle
 from dlio_benchmark.reader.reader_handler import FormatReader
 import tensorflow as tf
@@ -81,35 +81,33 @@ def _parse_image(self, serialized):
 
     @dlp.log
     def next(self):
-        logging.debug(
-            f"{utcnow()} Reading {len(self._file_list)} files thread {self.thread_index} rank {self._args.my_rank}")
-        self._dataset = tf.data.TFRecordDataset(filenames=self._file_list, buffer_size=self._args.transfer_size, 
+        logging.debug(f"{utcnow()} Reading {len(self._file_list)} files thread {self.thread_index} rank {self._args.my_rank}")
+        filenames = tf.data.Dataset.list_files(self._file_list, shuffle=True)
+        # sharding in the file list if we have enought files. 
+        if (len(self._file_list) >= self._args.comm_size):
+            filenames = filenames.shard(num_shards=self._args.comm_size, index=self._args.my_rank)
+
+        self._dataset = tf.data.TFRecordDataset(filenames=filenames, buffer_size=self._args.transfer_size,
                                                 num_parallel_reads=self._args.read_threads)
-
+				  
         if self._args.sample_shuffle != Shuffle.OFF:
             if self._args.sample_shuffle == Shuffle.SEED:
                 self._dataset = self._dataset.shuffle(buffer_size=self._args.shuffle_size,
                                           seed=self._args.seed)
             else:
                 self._dataset = self._dataset.shuffle(buffer_size=self._args.shuffle_size)
-
-        self._dataset = self._dataset.shard(num_shards=self._args.comm_size, index=self._args.my_rank)
-        if self._args.computation_threads==0:
-            self._dataset = self._dataset.batch(self.batch_size, drop_remainder=True)
-        else:
-            if self._args.computation_threads <= self.batch_size:
-                self._dataset = self._dataset.batch(self.batch_size, drop_remainder=True)
-                self._dataset = self._dataset.map(
-                    lambda x: tf.py_function(func=self._parse_image, inp=[x], Tout=[tf.uint8]), 
-                    num_parallel_calls=self._args.computation_threads)
-            else:
-                self._dataset = self._dataset.batch(self._args.computation_threads)
-                self._dataset = self._dataset.map(
-                    lambda x: tf.py_function(func=self._parse_image, inp=[x], Tout=[tf.uint8]), 
-                    num_parallel_calls=self._args.computation_threads)
-                self._dataset = self._dataset.unbatch(self.batch_size)
-        self._dataset = self._dataset.repeat(self._args.epochs)
-        total = math.ceil(len(self._file_list)/self._args.comm_size / self.batch_size * self._args.num_samples_per_file)
+
+        # shard the dataset if it is not done already.
+        if (len(self._file_list) < self._args.comm_size):
+            self._dataset =  self._dataset.shard(num_shards=self._args.comm_size, index=self._args.my_rank)
+
+        self._dataset = self._dataset.batch(self.batch_size, drop_remainder=True)
+        self._dataset = self._dataset.map(
+                lambda x: tf.py_function(func=self._parse_image, inp=[x], Tout=[tf.uint8]),
+                num_parallel_calls=self._args.computation_threads)
+
+        self._dataset = self._dataset.repeat()
+        total = math.floor(len(self._file_list)/self._args.comm_size / self.batch_size * self._args.num_samples_per_file)
         return self._dataset.take(total*self._args.epochs).prefetch(buffer_size=self._args.prefetch_size)
 
     @dlp.log

dlio_benchmark/storage/file_storage.py

@@ -24,7 +24,7 @@
 import glob
 import shutil
 
-from dlio_profiler.logger import fn_interceptor as Profile
+from dlio_benchmark.utils.utility import Profile
 
 dlp = Profile(MODULE_STORAGE)