From 2bcb0adf29d60eabf155a991260cbdd2c8c8c5b0 Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Wed, 2 Aug 2023 09:46:10 -0700 Subject: [PATCH 1/7] Clean up test settings --- src/kernel/Makefile | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/kernel/Makefile b/src/kernel/Makefile index 3a75fcc1..920d901d 100644 --- a/src/kernel/Makefile +++ b/src/kernel/Makefile @@ -857,25 +857,21 @@ help: first_test := 0 last_test := 999 -# Default threads. -ifeq ($(offload),1) -outer_threads := 2 -inner_threads := 2 -else -outer_threads := 8 -inner_threads := 2 -endif - TEST_MAKE_ARGS := real_bytes=8 use_rcp=0 allow_new_var_types=0 trace=1 TEST_MAKE := $(MAKE) $(TEST_MAKE_ARGS) -# Makefile functions for folding. +# Define makefile functions for folding. +# Set default threads. # Disable folding and checking for offload testing. ifeq ($(offload),1) FOLD = +outer_threads := 2 +inner_threads := 2 else -FOLD = fold=$(subst $(space),$(comma),$(1)) TEST_MAKE_ARGS += check=1 +FOLD = fold=$(subst $(space),$(comma),$(1)) +outer_threads := 8 +inner_threads := 2 endif ### Unit tests. From 92d5c719b567a69c1874a6ef2cb2c1b8c123092c Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Wed, 2 Aug 2023 09:46:46 -0700 Subject: [PATCH 2/7] Re-enable printing of current settings. --- src/common/common_utils.cpp | 2 +- src/compiler/compiler_main.cpp | 4 +-- src/kernel/yask_main.cpp | 58 +++++++++++++++++----------------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/common/common_utils.cpp b/src/common/common_utils.cpp index d0936ade..2acf7d5f 100644 --- a/src/common/common_utils.cpp +++ b/src/common/common_utils.cpp @@ -44,7 +44,7 @@ namespace yask { // for numbers above 9 (at least up to 99). // Format: "major.minor.patch[-alpha|-beta]". - const string version = "4.04.02"; + const string version = "4.04.03"; string yask_get_version_string() { return version; diff --git a/src/compiler/compiler_main.cpp b/src/compiler/compiler_main.cpp index f3e34a89..313df8bc 100644 --- a/src/compiler/compiler_main.cpp +++ b/src/compiler/compiler_main.cpp @@ -142,10 +142,10 @@ struct MySettings { // Show settings. ostringstream oss; - oss << "Options from the '" << pgm_name << "' binary:\n"; + oss << "Current settings from the '" << pgm_name << "' binary:\n"; parser.print_values(oss); auto cvals = csoln->get_command_line_values(); - oss << "Options from the YASK compiler library:\n" << + oss << "Current settings from the YASK compiler library:\n" << cvals; if (rem_args2.length()) diff --git a/src/kernel/yask_main.cpp b/src/kernel/yask_main.cpp index 41a2f312..28874210 100644 --- a/src/kernel/yask_main.cpp +++ b/src/kernel/yask_main.cpp @@ -157,19 +157,20 @@ struct MySettings { // Any remaining strings will be returned. auto rem_args = parser.parse_args(argc, argv); - // Handle additional knobs and help if there is a soln. + // Handle additional knobs and -help if there is a soln. if (ksoln) { + assert(kenv); + auto& os = kenv->get_debug_output()->get_ostream(); // Parse standard args not handled by this parser. rem_args = ksoln->apply_command_line_options(rem_args); if (help) { - cout << "Usage: " << pgm_name << " [options]\n" + os << "Usage: " << pgm_name << " [options]\n" "Options from the '" << pgm_name << "' binary:\n"; - parser.print_help(cout); - cout << "Options from the YASK library:\n" << - ksoln->get_command_line_help(); - cout << + parser.print_help(os); + os << "Options from the YASK library:\n" << + ksoln->get_command_line_help() << "\nValidation is very slow and uses 2x memory,\n" " so run with very small sizes and number of time-steps.\n" " If validation fails, it may be due to rounding error;\n" @@ -184,7 +185,7 @@ struct MySettings { exnr += " -nr" + dname + " " + to_string(i + 1); i++; } - cout << + os << "\nExamples:\n" " " << pgm_name << " -g 768 # global-domain size in all dims same.\n" " " << pgm_name << exg << " # global-domain size in each dim separately.\n" @@ -192,19 +193,24 @@ struct MySettings { " " << pgm_name << " -g 512" << exnr << " # number of ranks in each dim.\n" << " " << pgm_name << " -g 512" << exb << " -no-pre_auto_tune # manual block size.\n" << flush; - if (kenv) - kenv->exit(1); - else - exit(1); + kenv->exit(1); } - // Add settings. - ostringstream oss; - oss << "Options from the '" << pgm_name << "' binary:\n"; - parser.print_values(oss); - oss << "Options from the YASK library:\n" << + // Print splash banner and related info. + kenv->print_splash(argc, argv, "YASK Performance and Validation Utility invocation: "); + os << "\nStencil name: " << ksoln->get_name() << endl; + + // Print current settings. + os << "Current settings from the '" << pgm_name << "' binary:\n"; + parser.print_values(os); + os << "Current settings from the YASK library:\n" << ksoln->get_command_line_values(); + // Check option consistency. + kenv->assert_equality_over_ranks(num_trials, "number of trials"); + kenv->assert_equality_over_ranks(trial_steps, "number of steps per trial"); + kenv->assert_equality_over_ranks(validate ? 0 : 1, "validation"); + if (rem_args.length()) THROW_YASK_EXCEPTION("extraneous parameter(s): '" + rem_args + @@ -267,8 +273,13 @@ int main(int argc, char** argv) // Set up the environment. kenv = kfac.new_env(); + if (!kenv) + THROW_YASK_EXCEPTION("could not create YASK env"); auto num_ranks = kenv->get_num_ranks(); + // Make sure any MPI/OMP debug data is dumped from all ranks before continuing. + kenv->global_barrier(); + // Enable debug only on requested rank. if (opts.msg_rank != kenv->get_rank_index()) yk_env::disable_debug_output(); @@ -277,21 +288,10 @@ int main(int argc, char** argv) // Make solution object containing data and parameters for stencil eval. auto ksoln = kfac.new_solution(kenv); - // Parse custom and library-provided cmd-line options and - // exit on -help or error. + // Parse custom and library-provided cmd-line options, exit on -help + // or error, else show splash and current options. opts.parse(argc, argv, kenv, ksoln); - // Make sure any MPI/OMP debug data is dumped from all ranks before continuing - // and check option consistency. - kenv->global_barrier(); - kenv->assert_equality_over_ranks(opts.num_trials, "number of trials"); - kenv->assert_equality_over_ranks(opts.trial_steps, "number of steps per trial"); - kenv->assert_equality_over_ranks(opts.validate ? 0 : 1, "validation"); - - // Print splash banner and related info. - kenv->print_splash(argc, argv, "YASK Performance and Validation Utility invocation: "); - os << "\nStencil name: " << ksoln->get_name() << endl; - // Print PID and sleep for debug if needed. os << "\nPID: " << getpid() << endl; if (opts.debug_sleep) { From 15b617650a96ab2e3ac4c790b5be4c384f55d9bb Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Wed, 2 Aug 2023 09:46:46 -0700 Subject: [PATCH 3/7] Re-enable printing of current settings. --- src/common/common_utils.cpp | 2 +- src/compiler/compiler_main.cpp | 9 +++--- src/kernel/yask_main.cpp | 58 +++++++++++++++++----------------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/common/common_utils.cpp b/src/common/common_utils.cpp index d0936ade..2acf7d5f 100644 --- a/src/common/common_utils.cpp +++ b/src/common/common_utils.cpp @@ -44,7 +44,7 @@ namespace yask { // for numbers above 9 (at least up to 99). // Format: "major.minor.patch[-alpha|-beta]". - const string version = "4.04.02"; + const string version = "4.04.03"; string yask_get_version_string() { return version; diff --git a/src/compiler/compiler_main.cpp b/src/compiler/compiler_main.cpp index f3e34a89..3f5577fb 100644 --- a/src/compiler/compiler_main.cpp +++ b/src/compiler/compiler_main.cpp @@ -134,18 +134,17 @@ struct MySettings { cout << "\nExamples:\n" " " << pgm_name << " -stencil 3axis -radius 1 -target pseudo -p - # '-' for stdout\n" - " " << pgm_name << " -stencil awp -elem-bytes 8 -fold x=4,y=2 -target avx2 -p stencil_code.hpp\n" + " " << pgm_name << " -stencil awp -elem-bytes 8 -fold x=2,y=2 -target avx2 -p stencil_code.hpp\n" " " << pgm_name << " -stencil iso3dfd -radius 4 -target avx512 -p stencil_code.hpp\n" << flush; exit(1); } // Show settings. - ostringstream oss; - oss << "Options from the '" << pgm_name << "' binary:\n"; - parser.print_values(oss); + cout << "Current option settings from the '" << pgm_name << "' binary:\n"; + parser.print_values(cout); auto cvals = csoln->get_command_line_values(); - oss << "Options from the YASK compiler library:\n" << + cout << "Current option settings from the YASK compiler library:\n" << cvals; if (rem_args2.length()) diff --git a/src/kernel/yask_main.cpp b/src/kernel/yask_main.cpp index 41a2f312..6885e350 100644 --- a/src/kernel/yask_main.cpp +++ b/src/kernel/yask_main.cpp @@ -157,19 +157,20 @@ struct MySettings { // Any remaining strings will be returned. auto rem_args = parser.parse_args(argc, argv); - // Handle additional knobs and help if there is a soln. + // Handle additional knobs and -help if there is a soln. if (ksoln) { + assert(kenv); + auto& os = kenv->get_debug_output()->get_ostream(); // Parse standard args not handled by this parser. rem_args = ksoln->apply_command_line_options(rem_args); if (help) { - cout << "Usage: " << pgm_name << " [options]\n" + os << "Usage: " << pgm_name << " [options]\n" "Options from the '" << pgm_name << "' binary:\n"; - parser.print_help(cout); - cout << "Options from the YASK library:\n" << - ksoln->get_command_line_help(); - cout << + parser.print_help(os); + os << "Options from the YASK library:\n" << + ksoln->get_command_line_help() << "\nValidation is very slow and uses 2x memory,\n" " so run with very small sizes and number of time-steps.\n" " If validation fails, it may be due to rounding error;\n" @@ -184,7 +185,7 @@ struct MySettings { exnr += " -nr" + dname + " " + to_string(i + 1); i++; } - cout << + os << "\nExamples:\n" " " << pgm_name << " -g 768 # global-domain size in all dims same.\n" " " << pgm_name << exg << " # global-domain size in each dim separately.\n" @@ -192,19 +193,24 @@ struct MySettings { " " << pgm_name << " -g 512" << exnr << " # number of ranks in each dim.\n" << " " << pgm_name << " -g 512" << exb << " -no-pre_auto_tune # manual block size.\n" << flush; - if (kenv) - kenv->exit(1); - else - exit(1); + kenv->exit(1); } - // Add settings. - ostringstream oss; - oss << "Options from the '" << pgm_name << "' binary:\n"; - parser.print_values(oss); - oss << "Options from the YASK library:\n" << + // Print splash banner and related info. + kenv->print_splash(argc, argv, "YASK Performance and Validation Utility invocation: "); + os << "\nStencil name: " << ksoln->get_name() << endl; + + // Print current settings. + os << "Current option settings from the '" << pgm_name << "' binary:\n"; + parser.print_values(os); + os << "Current option settings from the YASK library:\n" << ksoln->get_command_line_values(); + // Check option consistency. + kenv->assert_equality_over_ranks(num_trials, "number of trials"); + kenv->assert_equality_over_ranks(trial_steps, "number of steps per trial"); + kenv->assert_equality_over_ranks(validate ? 0 : 1, "validation"); + if (rem_args.length()) THROW_YASK_EXCEPTION("extraneous parameter(s): '" + rem_args + @@ -267,8 +273,13 @@ int main(int argc, char** argv) // Set up the environment. kenv = kfac.new_env(); + if (!kenv) + THROW_YASK_EXCEPTION("could not create YASK env"); auto num_ranks = kenv->get_num_ranks(); + // Make sure any MPI/OMP debug data is dumped from all ranks before continuing. + kenv->global_barrier(); + // Enable debug only on requested rank. if (opts.msg_rank != kenv->get_rank_index()) yk_env::disable_debug_output(); @@ -277,21 +288,10 @@ int main(int argc, char** argv) // Make solution object containing data and parameters for stencil eval. auto ksoln = kfac.new_solution(kenv); - // Parse custom and library-provided cmd-line options and - // exit on -help or error. + // Parse custom and library-provided cmd-line options, exit on -help + // or error, else show splash and current options. opts.parse(argc, argv, kenv, ksoln); - // Make sure any MPI/OMP debug data is dumped from all ranks before continuing - // and check option consistency. - kenv->global_barrier(); - kenv->assert_equality_over_ranks(opts.num_trials, "number of trials"); - kenv->assert_equality_over_ranks(opts.trial_steps, "number of steps per trial"); - kenv->assert_equality_over_ranks(opts.validate ? 0 : 1, "validation"); - - // Print splash banner and related info. - kenv->print_splash(argc, argv, "YASK Performance and Validation Utility invocation: "); - os << "\nStencil name: " << ksoln->get_name() << endl; - // Print PID and sleep for debug if needed. os << "\nPID: " << getpid() << endl; if (opts.debug_sleep) { From a84b30331354729a239e37ea2bc394be3d1299ab Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Wed, 2 Aug 2023 16:27:54 -0700 Subject: [PATCH 4/7] Change default compiler settings for offload --- src/common/common.mk | 4 +--- src/kernel/Makefile | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/common/common.mk b/src/common/common.mk index 9c2f1fe4..7f5cafdf 100644 --- a/src/common/common.mk +++ b/src/common/common.mk @@ -190,9 +190,7 @@ ifeq ($(offload),1) endif # Base compiler flags for building kernel lib and apps. -ifeq ($(offload),1) - YK_CXXDBG := -gline-tables-only -else +ifeq ($(offload),0) YK_CXXDBG := -g endif YK_CXXOPT := -O3 diff --git a/src/kernel/Makefile b/src/kernel/Makefile index 920d901d..570a8ecd 100644 --- a/src/kernel/Makefile +++ b/src/kernel/Makefile @@ -42,6 +42,7 @@ use_ptrs ?= 1 use_safe_ptrs ?= 0 outer_domain_layout ?= 0 inner_misc_layout ?= 1 +first_inner ?= 1 early_loads ?= 0 min_buffer_len ?= 1 trace ?= 0 @@ -58,7 +59,6 @@ ifeq ($(offload),1) pfd_l1 := 0 pfd_l2 := 0 inner_loop_dim := 1 - early_loads := 1 outer_domain_layout := 1 endif @@ -190,6 +190,11 @@ ifeq ($(inner_misc_layout),1) else YC_FLAGS += -no-inner-misc-layout endif +ifeq ($(first_inner),1) + YC_FLAGS += -first-inner +else + YC_FLAGS += -no-first-inner +endif ifeq ($(early_loads),1) YC_FLAGS += -early-loads else @@ -346,8 +351,7 @@ endif # Compiler-specific settings. # Create a compiler invocation to test for macro settings. -# Use the flags defined so far. -YK_CXX_TEST := $(YK_CXX) $(YK_CXXFLAGS) +YK_CXX_TEST := $(YK_CXX) cxx_is_llvm_intel := $(call MACRO_DEF,$(YK_CXX_TEST),__INTEL_LLVM_COMPILER) cxx_is_clang := $(call MACRO_DEF,$(YK_CXX_TEST),__clang__) cxx_is_intel := $(call MACRO_DEF,$(YK_CXX_TEST),__INTEL_COMPILER) @@ -356,8 +360,8 @@ cxx_is_gnu := $(call MACRO_DEF,$(YK_CXX_TEST),__GNUC__) # LLVM-based Intel compiler (icpx). ifeq ($(cxx_is_llvm_intel),1) YK_CXXFLAGS2 += $(GXX_ISA) $(ICC_ISA) - ifeq ($(check),0) - YK_CXXDBG2 += -gline-tables-only + ifeq ($(offload),1) + YK_CXXDBG2 := -gline-tables-only endif YK_CXXDBG2 += -fdebug-info-for-profiling YK_CXXFLAGS2 += -ansi-alias \ @@ -763,8 +767,13 @@ echo-settings: echo YK_TAG=$(YK_TAG); \ echo YK_CXXVER=`$(YK_CXX) --version`; \ echo arch=$(arch); \ + echo cxx_is_llvm_intel=$(cxx_is_llvm_intel); \ + echo cxx_is_clang=$(cxx_is_clang); \ + echo cxx_is_intel=$(cxx_is_intel); \ + echo cxx_is_gnu=$(cxx_is_gnu); \ echo fold=$(fold); \ echo offload=$(offload); \ + echo offload_arch=$(offload_arch); \ echo offload_usm=$(offload_usm); \ echo pfd_l1=$(pfd_l1); \ echo pfd_l2=$(pfd_l2); \ From 3b351f4623b558b5251bb0b3fef09123c0b45cbc Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Wed, 2 Aug 2023 17:31:22 -0700 Subject: [PATCH 5/7] Clean up option setting printing. --- src/compiler/compiler_main.cpp | 17 ++++++++++------- src/kernel/yask_main.cpp | 4 ++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/compiler/compiler_main.cpp b/src/compiler/compiler_main.cpp index 3f5577fb..ca257428 100644 --- a/src/compiler/compiler_main.cpp +++ b/src/compiler/compiler_main.cpp @@ -80,7 +80,8 @@ struct MySettings { // Parse options from the command-line and set corresponding vars. // Exit with message on error or request for help. void parse(int argc, char** argv, - yc_solution_ptr csoln) { + yc_solution_ptr csoln, + bool print_info = true) { string pgm_name(argv[0]); string values; @@ -141,11 +142,13 @@ struct MySettings { } // Show settings. - cout << "Current option settings from the '" << pgm_name << "' binary:\n"; - parser.print_values(cout); - auto cvals = csoln->get_command_line_values(); - cout << "Current option settings from the YASK compiler library:\n" << - cvals; + if (print_info) { + cout << "Settings of options from the '" << pgm_name << "' binary:\n"; + parser.print_values(cout); + auto cvals = csoln->get_command_line_values(); + cout << "Settings of options from the YASK compiler library:\n" << + cvals; + } if (rem_args2.length()) THROW_YASK_EXCEPTION("extraneous parameter(s): '" + @@ -170,7 +173,7 @@ int main(int argc, char* argv[]) { // before the requested solution is chosen. { auto null_soln = factory.new_solution("temp"); - my_settings.parse(argc, argv, null_soln); + my_settings.parse(argc, argv, null_soln, false); } // Find the requested stencil in the registry. diff --git a/src/kernel/yask_main.cpp b/src/kernel/yask_main.cpp index 6885e350..5b43fa77 100644 --- a/src/kernel/yask_main.cpp +++ b/src/kernel/yask_main.cpp @@ -201,9 +201,9 @@ struct MySettings { os << "\nStencil name: " << ksoln->get_name() << endl; // Print current settings. - os << "Current option settings from the '" << pgm_name << "' binary:\n"; + os << "Settings of options from the '" << pgm_name << "' binary:\n"; parser.print_values(os); - os << "Current option settings from the YASK library:\n" << + os << "Settings of options from the YASK library:\n" << ksoln->get_command_line_values(); // Check option consistency. From a0a643072abb007fae407bfe4344f0f94eb52867 Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Thu, 3 Aug 2023 09:18:04 -0700 Subject: [PATCH 6/7] Update oneAPI version. Some other formatting. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 72974e2c..b0917186 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ and Intel(R) graphics processors. ### Pre-requisites: * Intel(R) [oneAPI](https://software.intel.com/content/www/us/en/develop/tools/oneapi.html) - HPC Toolkit for Linux (toolkit 2022.3 or later recommended; this will install - the Intel(R) oneAPI DPC++/C++ Compiler 2022.2.0). + HPC Toolkit for Linux (toolkit 2023.2 or later recommended) this will install + the Intel(R) oneAPI DPC++/C++ Compiler and the Intel(R) MPI Library. See notes below under version 4.00.00 changes. * Gnu C++ compiler, g++ (8.2.0 or later recommended). Even when using Intel compilers, a g++ installation is required. * Linux libraries `librt` and `libnuma`. * Grep. -* Perl (5.010 or later). +* Perl (v5 or later). * Awk. * Gnu make. * Bash shell. @@ -65,7 +65,7 @@ and Intel(R) graphics processors. ## Backward-compatibility notices ### Version 4 -* Version 4.04.00 deprecates the existing `void*` set/get_elements_in_slice() +* Version 4.04.00 deprecates the existing `void* {set,get}_elements_in_slice()` APIs and provides safer `float*` and `double*` versions. * Version 4.03.00 is a significant release with the following notices: - Each non-scratch stencil equation is now checked to ensure @@ -74,7 +74,7 @@ and Intel(R) graphics processors. (-1 is used for less-common reverse-time stencils.) - The `yk_solution::get_var()` API now throws an exception if the named var does not exist. (Used to return `std::nullptr`.) - - Vector clustering (unrolling by the YASK compiler) is no + - Vector "clustering" (unrolling by the YASK compiler) is no longer supported. - Read-ahead in the inner-loop is no longer supported. - APIs for getting OpenMP thread counts were added. From fd754e00f8207082cc1628e3a2b2161f11edfbed Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Thu, 3 Aug 2023 09:28:47 -0700 Subject: [PATCH 7/7] Gcc version update. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b0917186..0504391b 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,10 @@ and Intel(R) graphics processors. ### Pre-requisites: * Intel(R) [oneAPI](https://software.intel.com/content/www/us/en/develop/tools/oneapi.html) - HPC Toolkit for Linux (toolkit 2023.2 or later recommended) this will install + HPC Toolkit for Linux (toolkit 2023.2 or later recommended); this will install the Intel(R) oneAPI DPC++/C++ Compiler and the Intel(R) MPI Library. - See notes below under version 4.00.00 changes. -* Gnu C++ compiler, g++ (8.2.0 or later recommended). + See compiler notes below under version 4.00.00 changes. +* Gnu C++ compiler, g++ (8.5.0 or later recommended). Even when using Intel compilers, a g++ installation is required. * Linux libraries `librt` and `libnuma`. * Grep.