From b5b9ff7970d327cae8f7eea63e9cb5ea6e0a2237 Mon Sep 17 00:00:00 2001 From: Huihuo Zheng Date: Mon, 20 Nov 2023 16:43:38 -0700 Subject: [PATCH] Workloads (#112) * added support for multiprocessing context * added file access option for MPI IO * data loader optimzing * change profiling to false to disable iostat by default * added copyright info to python files that did not have it * fixed dali data loader execution. * fixing batch_size not defined issue * Merge branch 'main' into bugfix/dali-dl * fixes the calculation of image idx. * fixes the calculation of image idx. * added new workloads * set prefetch_factor to be 2 if it was set to be 0 * moved some part of the code init in tf_reader * fixed DataLoader args issues * recover tf_reader code * changed resnet50 tensorflow * fixed resnet50 config and sleep issue * fixed resnet50 config and sleep issue (#96) * added input x for compute * Workloads (#99) * fixed resnet50 config and sleep issue * added input x for compute * make it flexible in matching file patterns [e.g., jpeg==JPEG] * fixed typo * fixed fnmatch * fixed patterns matching case ignore * fixed bugs for tensorflow * fixed file list issue * renamed resnet50_tf * reflected recent changes of the DLIO-Profiler code * test-dlio-profiler * added dlio_profiler test * fixed test_dlio_profiler * removed test_dlio_profiler --------- Co-authored-by: Hariharan Devarajan --- dlio_benchmark/data_generator/tf_generator.py | 1 - dlio_benchmark/main.py | 1 - dlio_benchmark/reader/npz_reader.py | 2 +- dlio_benchmark/utils/utility.py | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dlio_benchmark/data_generator/tf_generator.py b/dlio_benchmark/data_generator/tf_generator.py index 60ccdfc8..f10a9621 100644 --- a/dlio_benchmark/data_generator/tf_generator.py +++ b/dlio_benchmark/data_generator/tf_generator.py @@ -18,7 +18,6 @@ from dlio_benchmark.data_generator.data_generator import DataGenerator import numpy as np import tensorflow as tf - from dlio_benchmark.utils.utility import progress, utcnow from dlio_profiler.logger import fn_interceptor as Profile from shutil import copyfile diff --git a/dlio_benchmark/main.py b/dlio_benchmark/main.py index 60a59312..8b5b0ec4 100644 --- a/dlio_benchmark/main.py +++ b/dlio_benchmark/main.py @@ -78,7 +78,6 @@ def __init__(self, cfg): self.output_folder = self.args.output_folder self.storage = StorageFactory().get_storage(self.args.storage_type, self.args.storage_root, self.args.framework) - self.output_folder = self.args.output_folder self.output = StorageFactory().get_storage(self.args.storage_type, self.args.output_folder, self.args.framework) diff --git a/dlio_benchmark/reader/npz_reader.py b/dlio_benchmark/reader/npz_reader.py index 7960d3bd..f0144f74 100644 --- a/dlio_benchmark/reader/npz_reader.py +++ b/dlio_benchmark/reader/npz_reader.py @@ -35,7 +35,7 @@ def __init__(self, dataset_type, thread_index, epoch): @dlp.log def open(self, filename): super().open(filename) - return np.load(filename, allow_pickle=True)["x"] + return np.load(filename, allow_pickle=True)['x'] @dlp.log def close(self, filename): diff --git a/dlio_benchmark/utils/utility.py b/dlio_benchmark/utils/utility.py index 047f877d..48690d43 100644 --- a/dlio_benchmark/utils/utility.py +++ b/dlio_benchmark/utils/utility.py @@ -158,4 +158,4 @@ def create_dur_event(name, cat, ts, dur, args={}): def get_trace_name(output_folder): - return f"{output_folder}/trace-{get_rank()}-of-{get_size()}.pfw" \ No newline at end of file + return f"{output_folder}/trace-{get_rank()}-of-{get_size()}.pfw"