From 1120cc25b84e2dfdf2509baaa67d4196f359088c Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Mon, 21 Nov 2022 16:20:36 -0800 Subject: [PATCH 1/3] Fix C++ kernel test to work on >1 rank. --- src/common/common_utils.cpp | 2 +- src/kernel/Makefile | 10 ++- src/kernel/lib/yk_var.hpp | 36 ++++++--- src/kernel/tests/yask_kernel_api_test.cpp | 97 +++++++++++++---------- 4 files changed, 87 insertions(+), 58 deletions(-) diff --git a/src/common/common_utils.cpp b/src/common/common_utils.cpp index 65d7085f..7ce586ad 100644 --- a/src/common/common_utils.cpp +++ b/src/common/common_utils.cpp @@ -43,7 +43,7 @@ namespace yask { // for numbers above 9 (at least up to 99). // Format: "major.minor.patch[-alpha|-beta]". - const string version = "4.00.02"; + const string version = "4.00.03"; string yask_get_version_string() { return version; diff --git a/src/kernel/Makefile b/src/kernel/Makefile index 05f801b6..38c83aa3 100644 --- a/src/kernel/Makefile +++ b/src/kernel/Makefile @@ -965,10 +965,12 @@ $(YK_API_TEST_EXEC): $(YK_TEST_SRC_DIR)/yask_kernel_api_test.cpp $(YK_LIB) @ls -l $@ # Build and run C++ tests. +# First run on 1 rank, then multiple ranks if ranks>1. cxx-yk-api-test: $(TEST_MAKE) $(YK_API_TEST_EXEC) allow_new_var_types=1 @echo '*** Running the C++ YASK kernel API test...' $(RUN_PREFIX) $(YK_API_TEST_EXEC) + if (( $(ranks) > 1 )); then $(RUN_PREFIX) mpirun -np $(ranks) $(YK_API_TEST_EXEC); fi # Run Python kernel API test. py-yk-api-test: $(YK_TEST_SRC_DIR)/yask_kernel_api_test.py $(YK_PY_LIB) @@ -1063,10 +1065,10 @@ yk-test-no-yc: kernel-no-yc $(YK_SCRIPT) # Run the kernel API tests for C++ and Python with and w/o expected exceptions. api-tests: - $(MAKE) clean; $(MAKE) cxx-yk-api-test real_bytes=8 stencil=iso3dfd - $(MAKE) clean; $(MAKE) py-yk-api-test stencil=iso3dfd - $(MAKE) clean; $(MAKE) cxx-yk-api-test-with-exception real_bytes=8 stencil=iso3dfd - $(MAKE) clean; $(MAKE) py-yk-api-test-with-exception stencil=iso3dfd + $(MAKE) clean; $(MAKE) cxx-yk-api-test real_bytes=8 + $(MAKE) clean; $(MAKE) py-yk-api-test + $(MAKE) clean; $(MAKE) cxx-yk-api-test-with-exception real_bytes=8 + $(MAKE) clean; $(MAKE) py-yk-api-test-with-exception # Run several stencils using built-in validation. # NB: set arch var as applicable. diff --git a/src/kernel/lib/yk_var.hpp b/src/kernel/lib/yk_var.hpp index 0582e5da..45733b1d 100644 --- a/src/kernel/lib/yk_var.hpp +++ b/src/kernel/lib/yk_var.hpp @@ -654,7 +654,11 @@ namespace yask { // Find sizes needed for slicing. inline Indices get_slice_range(const Indices& first_indices, const Indices& last_indices) const { - return last_indices.add_const(1).sub_elements(first_indices); + host_assert(first_indices.get_num_dims() == last_indices.get_num_dims()); + Indices range = first_indices; + for (int i = 0; i < first_indices.get_num_dims(); i++) + range[i] = std::max(idx_t(0), last_indices[i] + 1 - first_indices[i]); + return range; } // Accessors to GenericVar. @@ -955,8 +959,8 @@ namespace yask { template idx_t _visit_elements_in_slice(bool strict_indices, void* buffer_ptr, - const Indices& first_indices, - const Indices& last_indices, + const Indices& in_first_indices, + const Indices& in_last_indices, bool on_device) { STATE_VARS(this); if (get_storage() == 0) { @@ -969,11 +973,17 @@ namespace yask { } TRACE_MSG(Visitor::fname() << ": " << make_info_string() << " [" << - make_index_string(first_indices) << " ... " << - make_index_string(last_indices) << "] with buffer at " << + make_index_string(in_first_indices) << " ... " << + make_index_string(in_last_indices) << "] with buffer at " << buffer_ptr << " on " << (on_device ? "OMP device" : "host")); - check_indices(first_indices, Visitor::fname(), strict_indices, true, false); - check_indices(last_indices, Visitor::fname(), strict_indices, true, false); + Indices first_indices, last_indices; + check_indices(in_first_indices, Visitor::fname(), strict_indices, + true, false, &first_indices); + check_indices(in_last_indices, Visitor::fname(), strict_indices, + true, false, &last_indices); + TRACE_MSG(Visitor::fname() << ": clipped to [" << + make_index_string(first_indices) << " ... " << + make_index_string(last_indices) << "]"); // Find range. auto range = get_slice_range(first_indices, last_indices); @@ -1927,31 +1937,31 @@ namespace yask { } virtual idx_t set_element(double val, const Indices& indices, - bool strict_indices = false); + bool strict_indices); virtual idx_t set_element(double val, const VarIndices& indices, - bool strict_indices = false) { + bool strict_indices) { const Indices indices2(indices); return set_element(val, indices2, strict_indices); } virtual idx_t set_element(double val, const idx_t_init_list& indices, - bool strict_indices = false) { + bool strict_indices) { const Indices indices2(indices); return set_element(val, indices2, strict_indices); } virtual idx_t add_to_element(double val, const Indices& indices, - bool strict_indices = false); + bool strict_indices); virtual idx_t add_to_element(double val, const VarIndices& indices, - bool strict_indices = false) { + bool strict_indices) { const Indices indices2(indices); return add_to_element(val, indices2, strict_indices); } virtual idx_t add_to_element(double val, const idx_t_init_list& indices, - bool strict_indices = false) { + bool strict_indices) { const Indices indices2(indices); return add_to_element(val, indices2, strict_indices); } diff --git a/src/kernel/tests/yask_kernel_api_test.cpp b/src/kernel/tests/yask_kernel_api_test.cpp index 101cf43c..3a3fb7e4 100644 --- a/src/kernel/tests/yask_kernel_api_test.cpp +++ b/src/kernel/tests/yask_kernel_api_test.cpp @@ -36,23 +36,30 @@ IN THE SOFTWARE. using namespace std; using namespace yask; -int main() { - - // The factory from which all other kernel objects are made. - yk_factory kfac; - - // Initalize MPI, etc. - auto env = kfac.new_env(); +int main(int argc, char** argv) { + int rank_num = -1; try { + // The factory from which all other kernel objects are made. + yk_factory kfac; + + // Initalize MPI, etc. + auto env = kfac.new_env(); + rank_num = env->get_rank_index(); + + // Local options. + for (int i = 1; i < argc; i++) + if (string(argv[i]) == "-trace") + env->set_trace_enabled(true); + // Create solution. auto soln = kfac.new_solution(env); + // Apply any YASK command-line options. + soln->apply_command_line_options(argc, argv); + // Show output only from last rank. - // This is an example of using the rank APIs, - // the yask_output_factory, and set_debug_output(). - int rank_num = env->get_rank_index(); if (rank_num < env->get_num_ranks() - 1) { yk_env::disable_debug_output(); cout << "Suppressing output on rank " << rank_num << ".\n"; @@ -187,7 +194,7 @@ int main() { continue; // Create indices describing a subset of the overall domain. - vector first_indices, last_indices; + vector first_cube_indices, last_cube_indices; for (auto dname : var->get_dim_names()) { idx_t first_idx = 0, last_idx = 0; @@ -222,61 +229,69 @@ int main() { } // Add indices to index vectors. - first_indices.push_back(first_idx); - last_indices.push_back(last_idx); + first_cube_indices.push_back(first_idx); + last_cube_indices.push_back(last_idx); } - assert(first_indices.size() == ndims); - assert(last_indices.size() == ndims); + assert(first_cube_indices.size() == ndims); + assert(last_cube_indices.size() == ndims); // Print range of inner cube. if (ndims == 0) os << " inner-cube is a single point for this scalar var.\n"; else os << " range of inner-cube: " << - var->format_indices(first_indices) << " to " << - var->format_indices(last_indices) << ".\n"; + var->format_indices(first_cube_indices) << " to " << + var->format_indices(last_cube_indices) << ".\n"; // Check 2 corners of inner-cube to make sure they were set properly. - if (var->are_indices_local(first_indices)) { - auto val2 = var->get_element(first_indices); + if (var->are_indices_local(first_cube_indices)) { + auto val2 = var->get_element(first_cube_indices); os << " first element in inner cube == " << val2 << ".\n"; assert(val2 == outer_val); } else - os << " first element NOT in rank.\n"; - if (var->are_indices_local(last_indices)) { - auto val2 = var->get_element(last_indices); + os << " first element NOT in this rank.\n"; + if (var->are_indices_local(last_cube_indices)) { + auto val2 = var->get_element(last_cube_indices); os << " last element in inner cube == " << val2 << ".\n"; assert(val2 == outer_val); } else - os << " last element NOT in rank.\n"; + os << " last element NOT in this rank.\n"; // Re-init the values in the inner cube using the indices created above. + // Use 'strict_indices = false' because first/last_cube_indices are global, + // so part or all of cube may be outside this rank. double inner_val = 2.0; - bool strict_indices = false; // because first/last_indices are global. - idx_t nset = var->set_elements_in_slice_same(inner_val, first_indices, last_indices, strict_indices); + bool strict_indices = false; + idx_t nset = var->set_elements_in_slice_same(inner_val, first_cube_indices, + last_cube_indices, strict_indices); os << " " << nset << " element(s) in inner-cube set to " << inner_val << ".\n"; - // Check values on 2 corners of the inner cube to make sure they were set properly. - if (var->are_indices_local(first_indices)) { - auto val2 = var->get_element(first_indices); + // Check values on 2 corners of the inner cube to make sure they + // were set properly. Only check if element is in this rank. + if (var->are_indices_local(first_cube_indices)) { + auto val2 = var->get_element(first_cube_indices); os << " first element in inner cube == " << val2 << ".\n"; assert(val2 == inner_val); } - if (var->are_indices_local(last_indices)) { - auto val2 = var->get_element(last_indices); + if (var->are_indices_local(last_cube_indices)) { + auto val2 = var->get_element(last_cube_indices); os << " last element in inner cube == " << val2 << ".\n"; assert(val2 == inner_val); } - // Add to value in first corner of the inner cube if it's in this rank. - nset = var->add_to_element(1.0, first_indices); - os << " " << nset << " element(s) updated.\n"; - - // Check to make sure they were updated. - if (var->are_indices_local(first_indices)) { - auto val2 = var->get_element(first_indices); + // Add to value in first corner of the inner cube if it's in + // this rank. Here, we can use the default 'strict_indices = + // true' parameter to add_to_element() because the call is + // inside the check for local indices. + if (var->are_indices_local(first_cube_indices)) { + nset = var->add_to_element(1.0, first_cube_indices); + os << " " << nset << " element(s) updated.\n"; + assert(nset == 1); + + // Check to make sure they were updated. + auto val2 = var->get_element(first_cube_indices); os << " first element in inner cube == " << val2 << ".\n"; assert(val2 == inner_val + 1.0); } @@ -295,19 +310,21 @@ int main() { // Apply the stencil solution to the data. soln->reset_auto_tuner(false); env->global_barrier(); - os << "Running the solution for 1 step...\n"; + os << "Running the solution on " << env->get_num_ranks() << " rank(s)...\n"; + os << "Running for 1 step...\n"; soln->run_solution(0); - os << "Running the solution for 4 more steps...\n"; + os << "Running for 4 more steps...\n"; soln->run_solution(1, 4); soln->end_solution(); soln->get_stats(); + env->global_barrier(); os << "End of YASK kernel API test.\n"; return 0; } catch (yask_exception e) { cerr << "YASK kernel API test: " << e.get_message() << - " on rank " << env->get_rank_index() << ".\n"; + " on rank " << rank_num << ".\n"; return 1; } } From f8bfab2e82c95a2c97e8859d9406861318bdee84 Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Tue, 22 Nov 2022 11:34:44 -0800 Subject: [PATCH 2/3] Add yk_env::finalize() API. Call from yask_main.cpp and all API tests. --- .../html/classyask_1_1yk__env-members.html | 17 +++++---- docs/api/html/classyask_1_1yk__env.html | 31 ++++++++++++++++ docs/api/html/functions_f.html | 5 ++- docs/api/html/functions_func_f.html | 5 ++- docs/api/html/search/all_6.js | 1 + docs/api/html/search/functions_6.js | 1 + .../html/yask__kernel__api_8hpp_source.html | 7 ++-- docs/api/html/yk__var__api_8hpp_source.html | 2 +- include/yask_kernel_api.hpp | 12 ++++++ src/kernel/Makefile | 2 + src/kernel/lib/settings.hpp | 15 +++++++- src/kernel/lib/setup.cpp | 1 + src/kernel/tests/var_test.cpp | 1 + .../tests/yask_kernel_api_exception_test.cpp | 3 +- .../tests/yask_kernel_api_exception_test.py | 5 ++- src/kernel/tests/yask_kernel_api_test.cpp | 4 +- src/kernel/tests/yask_kernel_api_test.py | 37 ++++++++++++------- src/kernel/yask_main.cpp | 2 +- 18 files changed, 117 insertions(+), 34 deletions(-) diff --git a/docs/api/html/classyask_1_1yk__env-members.html b/docs/api/html/classyask_1_1yk__env-members.html index acc16f7c..84a18bc0 100644 --- a/docs/api/html/classyask_1_1yk__env-members.html +++ b/docs/api/html/classyask_1_1yk__env-members.html @@ -74,14 +74,15 @@

This is the complete list of members for yask::yk_env, including all inherited members.

- - - - - - - - + + + + + + + + +
disable_debug_output()yask::yk_envstatic
get_debug_output()yask::yk_envstatic
get_num_ranks() const =0yask::yk_envpure virtual
get_rank_index() const =0yask::yk_envpure virtual
global_barrier() const =0yask::yk_envpure virtual
is_trace_enabled()yask::yk_envstatic
set_debug_output(yask_output_ptr debug)yask::yk_envstatic
set_trace_enabled(bool enable)yask::yk_envstatic
~yk_env() (defined in yask::yk_env)yask::yk_envinlinevirtual
finalize()=0yask::yk_envpure virtual
get_debug_output()yask::yk_envstatic
get_num_ranks() const =0yask::yk_envpure virtual
get_rank_index() const =0yask::yk_envpure virtual
global_barrier() const =0yask::yk_envpure virtual
is_trace_enabled()yask::yk_envstatic
set_debug_output(yask_output_ptr debug)yask::yk_envstatic
set_trace_enabled(bool enable)yask::yk_envstatic
~yk_env() (defined in yask::yk_env)yask::yk_envinlinevirtual
diff --git a/docs/api/html/classyask_1_1yk__env.html b/docs/api/html/classyask_1_1yk__env.html index 7785359a..0589773a 100644 --- a/docs/api/html/classyask_1_1yk__env.html +++ b/docs/api/html/classyask_1_1yk__env.html @@ -91,6 +91,9 @@ virtual void global_barrier () const =0  Wait until all ranks have reached this element. More...
  +virtual void finalize ()=0 + Finalize the environment. More...
+  @@ -357,6 +360,34 @@

+

◆ finalize()

+ +
+
+

Static Public Member Functions

+ + + + +
+ + + + + + + +
virtual void yask::yk_env::finalize ()
+ 		+pure virtual
+
+ +

Finalize the environment.

+

If MPI is enabled and YASK initialized the MPI communicator, calls MPI_Finalize(). This function is automatically called when a yk_env object is destroyed. Cannot call global_barrier() or any MPI-dependent API after calling this.

+
The documentation for this class was generated from the following file:
    diff --git a/docs/api/html/functions_f.html b/docs/api/html/functions_f.html index ebd6ac62..9083f36f 100644 --- a/docs/api/html/functions_f.html +++ b/docs/api/html/functions_f.html @@ -65,6 +65,9 @@
    Here is a list of all documented class members with links to the class documentation for each member:

    - f -

      +
    • finalize() +: yask::yk_env +
    • first_domain_index() : yask::yc_solution_base
      • @@ -72,7 +75,7 @@

      - f -

        : yask::yc_solution
      • format_indices() -: yask::yk_var +: yask::yk_var
      • format_simple() : yask::yc_expr_node diff --git a/docs/api/html/functions_func_f.html b/docs/api/html/functions_func_f.html index b0d7aef0..a532d3ad 100644 --- a/docs/api/html/functions_func_f.html +++ b/docs/api/html/functions_func_f.html @@ -65,6 +65,9 @@  

        - f -

          +
        • finalize() +: yask::yk_env +
        • first_domain_index() : yask::yc_solution_base
          • @@ -72,7 +75,7 @@

          - f -

            : yask::yc_solution
          • format_indices() -: yask::yk_var +: yask::yk_var
          • format_simple() : yask::yc_expr_node diff --git a/docs/api/html/search/all_6.js b/docs/api/html/search/all_6.js index 8a6cb6d5..f9deef96 100644 --- a/docs/api/html/search/all_6.js +++ b/docs/api/html/search/all_6.js @@ -1,5 +1,6 @@ var searchData= [ + ['finalize',['finalize',['../classyask_1_1yk__env.html#ab2d17018fe9fadd9bfdc091d8bf32feb',1,'yask::yk_env']]], ['first_5fdomain_5findex',['first_domain_index',['../classyask_1_1yc__solution__base.html#a56f644e7e6b4e96619245f217b3763b5',1,'yask::yc_solution_base']]], ['format',['format',['../classyask_1_1yc__solution.html#a5f3624ded964c465724fd0221d5a2aaa',1,'yask::yc_solution']]], ['format_5findices',['format_indices',['../classyask_1_1yk__var.html#a55d6c585b8384881065c01bb067c96dd',1,'yask::yk_var::format_indices(const idx_t_vec &indices) const =0'],['../classyask_1_1yk__var.html#ae929dea5359d07e7541abab78ca7b139',1,'yask::yk_var::format_indices(const idx_t_init_list &indices) const =0']]], diff --git a/docs/api/html/search/functions_6.js b/docs/api/html/search/functions_6.js index 8a6cb6d5..f9deef96 100644 --- a/docs/api/html/search/functions_6.js +++ b/docs/api/html/search/functions_6.js @@ -1,5 +1,6 @@ var searchData= [ + ['finalize',['finalize',['../classyask_1_1yk__env.html#ab2d17018fe9fadd9bfdc091d8bf32feb',1,'yask::yk_env']]], ['first_5fdomain_5findex',['first_domain_index',['../classyask_1_1yc__solution__base.html#a56f644e7e6b4e96619245f217b3763b5',1,'yask::yc_solution_base']]], ['format',['format',['../classyask_1_1yc__solution.html#a5f3624ded964c465724fd0221d5a2aaa',1,'yask::yc_solution']]], ['format_5findices',['format_indices',['../classyask_1_1yk__var.html#a55d6c585b8384881065c01bb067c96dd',1,'yask::yk_var::format_indices(const idx_t_vec &indices) const =0'],['../classyask_1_1yk__var.html#ae929dea5359d07e7541abab78ca7b139',1,'yask::yk_var::format_indices(const idx_t_init_list &indices) const =0']]], diff --git a/docs/api/html/yask__kernel__api_8hpp_source.html b/docs/api/html/yask__kernel__api_8hpp_source.html index 5a6e4d06..1b700cc4 100644 --- a/docs/api/html/yask__kernel__api_8hpp_source.html +++ b/docs/api/html/yask__kernel__api_8hpp_source.html @@ -70,17 +70,18 @@
            yask_kernel_api.hpp
            -Go to the documentation of this file.
            1 /*****************************************************************************
            2 
            3 YASK: Yet Another Stencil Kit
            4 Copyright (c) 2014-2022, Intel Corporation
            5 
            6 Permission is hereby granted, free of charge, to any person obtaining a copy
            7 of this software and associated documentation files (the "Software"), to
            8 deal in the Software without restriction, including without limitation the
            9 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
            10 sell copies of the Software, and to permit persons to whom the Software is
            11 furnished to do so, subject to the following conditions:
            12 
            13 * The above copyright notice and this permission notice shall be included in
            14  all copies or substantial portions of the Software.
            15 
            16 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
            17 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
            18 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
            19 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
            20 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
            21 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
            22 IN THE SOFTWARE.
            23 
            24 *****************************************************************************/
            25 
            27 
            28 // This file uses Doxygen 1.8 markup for API documentation-generation.
            29 // See http://www.stack.nl/~dimitri/doxygen.
            32 #pragma once
            33 
            34 #include "yask_common_api.hpp"
            35 #include <vector>
            36 #include <cinttypes>
            37 
            38 #ifndef MPI_VERSION
            39 typedef int MPI_Comm;
            40 #endif
            41 
            42 namespace yask {
            43 
            50  // Forward declarations of classes and pointers.
            51 
            52  class yk_env;
            54  typedef std::shared_ptr<yk_env> yk_env_ptr;
            55 
            56  class yk_solution;
            58  typedef std::shared_ptr<yk_solution> yk_solution_ptr;
            59 
            60  class yk_var;
            62  typedef std::shared_ptr<yk_var> yk_var_ptr;
            63 
            64  class yk_stats;
            66  typedef std::shared_ptr<yk_stats> yk_stats_ptr;
            67 
            69 } // namespace yask.
            70 
            71 #include "aux/yk_solution_api.hpp"
            72 #include "aux/yk_var_api.hpp"
            73 
            74 namespace yask {
            75 
            81  class yk_factory {
            83  public:
            84  yk_factory();
            85  virtual ~yk_factory() {}
            86 
            88 
            91  virtual std::string
            92  get_version_string();
            93 
            95 
            123  virtual yk_env_ptr
            124  new_env() const;
            125 
            127 
            136  virtual yk_env_ptr
            137  new_env(MPI_Comm comm) const;
            138 
            140 
            145  virtual yk_solution_ptr
            146  new_solution(yk_env_ptr env ) const;
            147 
            149 
            156  virtual yk_solution_ptr
            157  new_solution(yk_env_ptr env ,
            158  const yk_solution_ptr source ) const;
            161  }; // yk_factory.
            162 
            164 
            167  class yk_env {
            168  public:
            169  virtual ~yk_env() {}
            170 
            172 
            177  static void
            178  set_debug_output(yask_output_ptr debug );
            181 
            183 
            186  static void
            187  disable_debug_output();
            188 
            190 
            197  static yask_output_ptr
            198  get_debug_output();
            199 
            201 
            208  static void
            209  set_trace_enabled(bool enable);
            210 
            212 
            218  static bool
            219  is_trace_enabled();
            220 
            222 
            225  virtual int get_num_ranks() const =0;
            226 
            228 
            231  virtual int get_rank_index() const =0;
            232 
            234 
            238  virtual void
            239  global_barrier() const =0;
            240 
            241  }; // yk_env.
            242 
            244  YASK_DEPRECATED
            245  typedef yk_var yk_grid;
            247  YASK_DEPRECATED
            248  typedef yk_var_ptr yk_grid_ptr;
            249 
            252 } // namespace yask.
            yask::yk_env::disable_debug_output
            static void disable_debug_output()
            Disable the debug output.
            +Go to the documentation of this file.
            1 /*****************************************************************************
            2 
            3 YASK: Yet Another Stencil Kit
            4 Copyright (c) 2014-2022, Intel Corporation
            5 
            6 Permission is hereby granted, free of charge, to any person obtaining a copy
            7 of this software and associated documentation files (the "Software"), to
            8 deal in the Software without restriction, including without limitation the
            9 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
            10 sell copies of the Software, and to permit persons to whom the Software is
            11 furnished to do so, subject to the following conditions:
            12 
            13 * The above copyright notice and this permission notice shall be included in
            14  all copies or substantial portions of the Software.
            15 
            16 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
            17 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
            18 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
            19 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
            20 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
            21 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
            22 IN THE SOFTWARE.
            23 
            24 *****************************************************************************/
            25 
            27 
            28 // This file uses Doxygen 1.8 markup for API documentation-generation.
            29 // See http://www.stack.nl/~dimitri/doxygen.
            32 #pragma once
            33 
            34 #include "yask_common_api.hpp"
            35 #include <vector>
            36 #include <cinttypes>
            37 
            38 #ifndef MPI_VERSION
            39 typedef int MPI_Comm;
            40 #endif
            41 
            42 namespace yask {
            43 
            50  // Forward declarations of classes and pointers.
            51 
            52  class yk_env;
            54  typedef std::shared_ptr<yk_env> yk_env_ptr;
            55 
            56  class yk_solution;
            58  typedef std::shared_ptr<yk_solution> yk_solution_ptr;
            59 
            60  class yk_var;
            62  typedef std::shared_ptr<yk_var> yk_var_ptr;
            63 
            64  class yk_stats;
            66  typedef std::shared_ptr<yk_stats> yk_stats_ptr;
            67 
            69 } // namespace yask.
            70 
            71 #include "aux/yk_solution_api.hpp"
            72 #include "aux/yk_var_api.hpp"
            73 
            74 namespace yask {
            75 
            81  class yk_factory {
            83  public:
            84  yk_factory();
            85  virtual ~yk_factory() {}
            86 
            88 
            91  virtual std::string
            92  get_version_string();
            93 
            95 
            123  virtual yk_env_ptr
            124  new_env() const;
            125 
            127 
            136  virtual yk_env_ptr
            137  new_env(MPI_Comm comm) const;
            138 
            140 
            145  virtual yk_solution_ptr
            146  new_solution(yk_env_ptr env ) const;
            147 
            149 
            156  virtual yk_solution_ptr
            157  new_solution(yk_env_ptr env ,
            158  const yk_solution_ptr source ) const;
            161  }; // yk_factory.
            162 
            164 
            167  class yk_env {
            168  public:
            169  virtual ~yk_env() {}
            170 
            172 
            177  static void
            178  set_debug_output(yask_output_ptr debug );
            181 
            183 
            186  static void
            187  disable_debug_output();
            188 
            190 
            197  static yask_output_ptr
            198  get_debug_output();
            199 
            201 
            208  static void
            209  set_trace_enabled(bool enable);
            210 
            212 
            218  static bool
            219  is_trace_enabled();
            220 
            222 
            225  virtual int get_num_ranks() const =0;
            226 
            228 
            231  virtual int get_rank_index() const =0;
            232 
            234 
            238  virtual void
            239  global_barrier() const =0;
            240 
            242 
            250  virtual void
            251  finalize() =0;
            252 
            253  }; // yk_env.
            254 
            256  YASK_DEPRECATED
            257  typedef yk_var yk_grid;
            259  YASK_DEPRECATED
            260  typedef yk_var_ptr yk_grid_ptr;
            261 
            264 } // namespace yask.
            yask::yk_env::disable_debug_output
            static void disable_debug_output()
            Disable the debug output.
            yask::yk_stats
            Statistics from calls to run_solution().
            Definition: yk_solution_api.hpp:1219
            yask::yk_stats_ptr
            std::shared_ptr< yk_stats > yk_stats_ptr
            Shared pointer to yk_stats.
            Definition: yask_kernel_api.hpp:64
            yask::yk_env
            Kernel environment.
            Definition: yask_kernel_api.hpp:167
            yask::yk_factory::new_env
            virtual yk_env_ptr new_env() const
            Create an object to hold environment information.
            yask::yk_env::is_trace_enabled
            static bool is_trace_enabled()
            Get whether tracing is enabled.
            yask::yk_env::get_num_ranks
            virtual int get_num_ranks() const =0
            Get number of MPI ranks.
            +
            yask::yk_env::finalize
            virtual void finalize()=0
            Finalize the environment.
            yask::yk_env::get_debug_output
            static yask_output_ptr get_debug_output()
            Get object to receive debug output.
            yask::yk_env_ptr
            std::shared_ptr< yk_env > yk_env_ptr
            Shared pointer to yk_env.
            Definition: yask_kernel_api.hpp:52
            yk_solution_api.hpp
            -
            yask::yk_grid_ptr
            YASK_DEPRECATED typedef yk_var_ptr yk_grid_ptr
            [Deprecated] Use yk_var_ptr.
            Definition: yask_kernel_api.hpp:248
            +
            yask::yk_grid_ptr
            YASK_DEPRECATED typedef yk_var_ptr yk_grid_ptr
            [Deprecated] Use yk_var_ptr.
            Definition: yask_kernel_api.hpp:260
            yask::yk_factory::get_version_string
            virtual std::string get_version_string()
            Version information.
            yask::yask_output_ptr
            std::shared_ptr< yask_output > yask_output_ptr
            Shared pointer to yask_output.
            Definition: yask_common_api.hpp:94
            yask::yk_var
            A run-time YASK data container.
            Definition: yk_var_api.hpp:185
            @@ -93,7 +94,7 @@
            yask::yk_solution_ptr
            std::shared_ptr< yk_solution > yk_solution_ptr
            Shared pointer to yk_solution.
            Definition: yask_kernel_api.hpp:56
            yask::yk_env::set_debug_output
            static void set_debug_output(yask_output_ptr debug)
            Set object to receive debug output.
            yask::yk_solution
            Stencil solution as defined by the generated code from the YASK stencil compiler.
            Definition: yk_solution_api.hpp:82
            -
            yask::yk_grid
            YASK_DEPRECATED typedef yk_var yk_grid
            [Deprecated] Use yk_var.
            Definition: yask_kernel_api.hpp:245
            +
            yask::yk_grid
            YASK_DEPRECATED typedef yk_var yk_grid
            [Deprecated] Use yk_var.
            Definition: yask_kernel_api.hpp:257
            yk_var_api.hpp
            yask::yk_var_ptr
            std::shared_ptr< yk_var > yk_var_ptr
            Shared pointer to yk_var.
            Definition: yask_kernel_api.hpp:60
            diff --git a/docs/api/html/yk__var__api_8hpp_source.html b/docs/api/html/yk__var__api_8hpp_source.html index 0d091f95..1080a972 100644 --- a/docs/api/html/yk__var__api_8hpp_source.html +++ b/docs/api/html/yk__var__api_8hpp_source.html @@ -132,7 +132,7 @@
            yask_kernel_api.hpp
            yask::yk_var::is_fixed_size
            virtual bool is_fixed_size() const =0
            Determine whether this var is not automatically resized based on the solution.
            yask::yk_var::get_raw_storage_buffer
            virtual void * get_raw_storage_buffer()=0
            [Advanced] Get pointer to raw data storage buffer.
            -
            yask::yk_grid
            YASK_DEPRECATED typedef yk_var yk_grid
            [Deprecated] Use yk_var.
            Definition: yask_kernel_api.hpp:245
            +
            yask::yk_grid
            YASK_DEPRECATED typedef yk_var yk_grid
            [Deprecated] Use yk_var.
            Definition: yask_kernel_api.hpp:257
            yask::yk_var::get_left_extra_pad_size
            virtual idx_t get_left_extra_pad_size(const std::string &dim) const =0
            Get the actual extra left padding in the specified dimension.
            yask::yk_var::is_storage_layout_identical
            virtual bool is_storage_layout_identical(const yk_var_ptr other) const =0
            [Advanced] Determines whether storage layout is the same as another var.
            yask::yk_var::get_first_local_index
            virtual idx_t get_first_local_index(const std::string &dim) const =0
            Get the first valid index in this rank in the specified dimension.
            diff --git a/include/yask_kernel_api.hpp b/include/yask_kernel_api.hpp index 42e78ba4..f0d493ba 100644 --- a/include/yask_kernel_api.hpp +++ b/include/yask_kernel_api.hpp @@ -238,6 +238,18 @@ namespace yask { virtual void global_barrier() const =0; + /// Finalize the environment. + /** + If MPI is enabled and YASK initialized the MPI communicator, + calls `MPI_Finalize()`. + This function is automatically called when a yk_env object + is destroyed. + Cannot call global_barrier() or any MPI-dependent API after + calling this. + */ + virtual void + finalize() =0; + }; // yk_env. /// **[Deprecated]** Use yk_var. diff --git a/src/kernel/Makefile b/src/kernel/Makefile index 38c83aa3..b6f66673 100644 --- a/src/kernel/Makefile +++ b/src/kernel/Makefile @@ -973,9 +973,11 @@ cxx-yk-api-test: if (( $(ranks) > 1 )); then $(RUN_PREFIX) mpirun -np $(ranks) $(YK_API_TEST_EXEC); fi # Run Python kernel API test. +# First run on 1 rank, then multiple ranks if ranks>1. py-yk-api-test: $(YK_TEST_SRC_DIR)/yask_kernel_api_test.py $(YK_PY_LIB) @echo '*** Running the Python YASK kernel API test...' $(RUN_PYTHON) $< + if (( $(ranks) > 1 )); then mpirun -np $(ranks) $(RUN_PYTHON) $<; fi # Build C++ kernel tests with exceptions. $(YK_API_TEST_EXEC_WITH_EXCEPTION): $(YK_TEST_SRC_DIR)/yask_kernel_api_exception_test.cpp $(YK_LIB) diff --git a/src/kernel/lib/settings.hpp b/src/kernel/lib/settings.hpp index 498e3b1f..fdf7c495 100644 --- a/src/kernel/lib/settings.hpp +++ b/src/kernel/lib/settings.hpp @@ -70,6 +70,7 @@ namespace yask { MPI_Group group = MPI_GROUP_NULL; int num_ranks = 1; // total number of ranks. int my_rank = 0; // MPI-assigned index. + bool finalize_needed = false; // Vars for shared-mem ranks. MPI_Comm shm_comm = MPI_COMM_NULL; // shm communicator. @@ -81,12 +82,24 @@ namespace yask { int max_threads=0; // initial value from OMP. KernelEnv() { } - virtual ~KernelEnv() { } + virtual ~KernelEnv() { + finalize(); + } // Init MPI, OMP, etc. // This is normally called very early in the program. virtual void init_env(int* argc, char*** argv, MPI_Comm comm); + virtual void finalize() { + TRACE_MSG("finalize_needed = " << finalize_needed); + if (comm != MPI_COMM_NULL && finalize_needed) { + MPI_Finalize(); + comm = MPI_COMM_NULL; + shm_comm = MPI_COMM_NULL; + } + finalize_needed = false; + } + // Lock. static void set_debug_lock() { if (!_debug_lock_init_done) { diff --git a/src/kernel/lib/setup.cpp b/src/kernel/lib/setup.cpp index 5ed9e1c2..68ce154f 100644 --- a/src/kernel/lib/setup.cpp +++ b/src/kernel/lib/setup.cpp @@ -61,6 +61,7 @@ namespace yask { THROW_YASK_EXCEPTION("error: MPI_THREAD_SERIALIZED or MPI_THREAD_MULTIPLE not provided"); } is_init = true; + finalize_needed = true; } comm = MPI_COMM_WORLD; } diff --git a/src/kernel/tests/var_test.cpp b/src/kernel/tests/var_test.cpp index 300d8c5c..edbb9876 100644 --- a/src/kernel/tests/var_test.cpp +++ b/src/kernel/tests/var_test.cpp @@ -263,6 +263,7 @@ void run_tests(int argc, char* argv[]) { offload_map_free(buf, sz); os << "Exiting 3-D test\n"; } + kenv->finalize(); } int main(int argc, char* argv[]) { diff --git a/src/kernel/tests/yask_kernel_api_exception_test.cpp b/src/kernel/tests/yask_kernel_api_exception_test.cpp index dbea63a3..e8f1cddb 100644 --- a/src/kernel/tests/yask_kernel_api_exception_test.cpp +++ b/src/kernel/tests/yask_kernel_api_exception_test.cpp @@ -99,6 +99,7 @@ int main() { soln->run_solution(1, 10); soln->end_solution(); soln->get_stats(); + env->finalize(); // TODO: add exception tests for the methods below: // StencilContext::calc_region @@ -128,7 +129,7 @@ int main() { exit(1); } else - cout << "End of YASK kernel API test with exception.\n"; + cout << "End of YASK kernel C++ API test with exception.\n"; return 0; } diff --git a/src/kernel/tests/yask_kernel_api_exception_test.py b/src/kernel/tests/yask_kernel_api_exception_test.py index 639c3fee..d8a135ce 100755 --- a/src/kernel/tests/yask_kernel_api_exception_test.py +++ b/src/kernel/tests/yask_kernel_api_exception_test.py @@ -105,9 +105,10 @@ soln.end_solution() soln.get_stats() #print("Debug output captured:\n", debug_output.get_string()) - + env.finalize() + if num_exception != 2: print("There is a problem in exception test.") exit(1) else: - print("End of YASK kernel API test with exception.") + print("End of YASK Python kernel API test with exception.") diff --git a/src/kernel/tests/yask_kernel_api_test.cpp b/src/kernel/tests/yask_kernel_api_test.cpp index 3a3fb7e4..eac9ffb4 100644 --- a/src/kernel/tests/yask_kernel_api_test.cpp +++ b/src/kernel/tests/yask_kernel_api_test.cpp @@ -318,8 +318,8 @@ int main(int argc, char** argv) { soln->end_solution(); soln->get_stats(); - env->global_barrier(); - os << "End of YASK kernel API test.\n"; + env->finalize(); + os << "End of YASK C++ kernel API test.\n"; return 0; } catch (yask_exception e) { diff --git a/src/kernel/tests/yask_kernel_api_test.py b/src/kernel/tests/yask_kernel_api_test.py index 32f796ee..01f92074 100755 --- a/src/kernel/tests/yask_kernel_api_test.py +++ b/src/kernel/tests/yask_kernel_api_test.py @@ -86,11 +86,13 @@ def read_var(var, timestep) : print("Testing reading var '", var.get_name(), "' at time ", timestep, "...") ndarray, first_indices, last_indices, point = make_ndarray(var, timestep) - print("Reading 1 element...") + print("Reading corner elements...") val1 = var.get_element(first_indices) print("Read value ", val1) + val2 = var.get_element(last_indices) + print("Read value ", val2) - print("Reading all element(s) in ndarray...") + print("Reading element(s) in ndarray from ", first_indices, " to ", last_indices, "...") nread = var.get_elements_in_slice(ndarray.data, first_indices, last_indices) print(ndarray) @@ -159,6 +161,10 @@ def init_var(var, timestep) : # Initalize MPI, etc. env = kfac.new_env() + if env.get_rank_index() > 0: + env.disable_debug_output(); + else: + env.set_trace_enabled(True); # Create solution. soln = kfac.new_solution(env) @@ -208,24 +214,27 @@ def init_var(var, timestep) : # Print some info about the solution. print("Stencil-solution '", name, "':") - print(" Step dimension: ", soln.get_step_dim_name()) - print(" Domain dimensions: ", soln.get_domain_dim_names()) + print(" Step dimension:", soln.get_step_dim_name()) + print(" Domain dimensions:", soln.get_domain_dim_names()) print(" Vars:") for var in soln.get_vars() : print(" ", var.get_name(), var.get_dim_names()) for dname in var.get_dim_names() : if dname in soln.get_domain_dim_names() : print(" '", dname, "' allowed domain index range in this rank: ", - var.get_first_rank_alloc_index(dname), " ... ", + var.get_first_rank_alloc_index(dname), "...", var.get_last_rank_alloc_index(dname)) elif dname == soln.get_step_dim_name() : print(" '", dname, "' allowed step index range: ", - var.get_first_valid_step_index(), " ... ", + var.get_first_valid_step_index(), "...", var.get_last_valid_step_index()) else : print(" '", dname, "' allowed misc index range: ", - var.get_first_misc_index(dname), " ... ", + var.get_first_misc_index(dname), "...", var.get_last_misc_index(dname)) + print(" allowed range in all dims: ", + var.get_first_local_index_vec(), "...", + var.get_last_local_index_vec()) # Init the vars. for var in soln.get_vars() : @@ -292,7 +301,7 @@ def init_var(var, timestep) : last_indices += [var.get_last_misc_index(dname)] # Init value at one point. - nset = var.set_element(15.0, one_indices) + nset = var.set_element(15.0, one_indices, False) print("Set ", nset, " element(s) in rank ", env.get_rank_index()) # Init the values within the small cube. @@ -304,22 +313,24 @@ def init_var(var, timestep) : # Apply the stencil solution to the data. env.global_barrier() - print("Running the solution for 1 step...") + print("Running the solution on", env.get_num_ranks(), "rank(s)"); + print("Running for 1 step...") soln.run_solution(0) # Print result at timestep 1. for var in soln.get_vars() : read_var(var, 1) - print("Running the solution for 10 more steps...") - soln.run_solution(1, 10) + print("Running for 4 more steps...") + soln.run_solution(1, 4) - # Print final result at timestep 11, assuming update was to t+1. + # Print final result at timestep 5, assuming last update was to t+1. for var in soln.get_vars() : - read_var(var, 11) + read_var(var, 5) soln.end_solution() soln.get_stats() + env.finalize() #print("Debug output captured:\n", debug_output.get_string()) print("End of YASK Python kernel API test.") diff --git a/src/kernel/yask_main.cpp b/src/kernel/yask_main.cpp index b48b0585..4515d771 100644 --- a/src/kernel/yask_main.cpp +++ b/src/kernel/yask_main.cpp @@ -652,7 +652,7 @@ int main(int argc, char** argv) if (!ok) exit_yask(1); - MPI_Finalize(); + kenv->finalize(); os << "YASK DONE." << endl << div_line << flush; } catch (yask_exception& e) { From dfadd644360344353a17be321c6931eb78849c8c Mon Sep 17 00:00:00 2001 From: "Yount, Chuck" Date: Tue, 22 Nov 2022 11:42:30 -0800 Subject: [PATCH 3/3] Fix var unit test. --- src/kernel/tests/var_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kernel/tests/var_test.cpp b/src/kernel/tests/var_test.cpp index edbb9876..ee599164 100644 --- a/src/kernel/tests/var_test.cpp +++ b/src/kernel/tests/var_test.cpp @@ -87,8 +87,8 @@ void run_tests(int argc, char* argv[]) { double val = 3.14; os << "Testing with " << val << endl; - g0->set_element(val, {}); - g1->set_element(val, {}); + g0->set_element(val, {}, true); + g1->set_element(val, {}, true); auto v0 = g0->get_element({}); auto v1 = g1->get_element({}); assert(v0 == v1); @@ -133,8 +133,8 @@ void run_tests(int argc, char* argv[]) { IdxTuple first, last; for (auto dname : gdims) { - first.add_dim_back(dname, g3->get_first_rank_alloc_index(dname)); - last.add_dim_back(dname, g3->get_last_rank_alloc_index(dname)); + first.add_dim_back(dname, g3->get_first_local_index(dname)); + last.add_dim_back(dname, g3->get_last_local_index(dname)); } Indices firsti(first), lasti(last);