From 4df015372bc1586f40679768a1567d11b7374758 Mon Sep 17 00:00:00 2001 From: Trevor Steil Date: Thu, 1 Aug 2024 11:31:28 -0700 Subject: [PATCH 01/40] Fixes bug in comm::barrier() that can cause asyncs to be spawned from a rank that has completed a barrier() and processed on a different rank that has not completed the same barrier() (#224) --- include/ygm/detail/comm.ipp | 2 ++ test/CMakeLists.txt | 1 + test/test_barrier.cpp | 28 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 test/test_barrier.cpp diff --git a/include/ygm/detail/comm.ipp b/include/ygm/detail/comm.ipp index e123e81c..7cdd8d63 100644 --- a/include/ygm/detail/comm.ipp +++ b/include/ygm/detail/comm.ipp @@ -244,6 +244,8 @@ inline void comm::barrier() { } ASSERT_RELEASE(m_pre_barrier_callbacks.empty()); ASSERT_RELEASE(m_send_dest_queue.empty()); + + cf_barrier(); } /** diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7dc708c8..01c35a9f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,6 +35,7 @@ add_ygm_seq_test(test_cereal_archive) add_ygm_test(test_comm) add_ygm_test(test_comm_2) +add_ygm_test(test_barrier) add_ygm_test(test_layout) add_ygm_test(test_large_messages) add_ygm_test(test_map) diff --git a/test/test_barrier.cpp b/test/test_barrier.cpp new file mode 100644 index 00000000..62fc1cae --- /dev/null +++ b/test/test_barrier.cpp @@ -0,0 +1,28 @@ +// Copyright 2019-2021 Lawrence Livermore National Security, LLC and other YGM +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: MIT + +#undef NDEBUG + +#include + +int main(int argc, char **argv) { + ygm::comm world(&argc, &argv); + + // Test barriers for early exit + { + int num_rounds = 100; + static int round = 0; + for (int i = 0; i < num_rounds; ++i) { + world.async_bcast( + [](int curr_round) { ASSERT_RELEASE(curr_round == round); }, round); + + world.barrier(); + + ++round; + } + } + + return 0; +} From d82d41ca1ec1c52a9874226486767f5dd426d3a5 Mon Sep 17 00:00:00 2001 From: Trevor Steil Date: Thu, 1 Aug 2024 11:32:33 -0700 Subject: [PATCH 02/40] Fixes bug in block_partitioner (#225) --- include/ygm/container/array.hpp | 14 +++++++------- include/ygm/container/detail/block_partitioner.hpp | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/ygm/container/array.hpp b/include/ygm/container/array.hpp index 14e11bae..49b19538 100644 --- a/include/ygm/container/array.hpp +++ b/include/ygm/container/array.hpp @@ -355,8 +355,10 @@ class array std::vector> tmp_values; tmp_values.reserve(local_size()); local_for_all( - [&tmp_values](const key_type& index, const mapped_type& value) { - tmp_values.push_back(std::make_pair(index, value)); + [&tmp_values, size](const key_type& index, const mapped_type& value) { + if (index < size) { + tmp_values.push_back(std::make_pair(index, value)); + } }); m_global_size = size; @@ -368,14 +370,14 @@ class array // Repopulate array values for (const auto& [index, value] : tmp_values) { - if (index < size) { - async_set(index, value); - } + async_set(index, value); } m_comm.barrier(); } + void resize(const size_type size) { resize(size, m_default_value); } + size_t local_size() { return partitioner.local_size(); } size_t size() const { @@ -383,8 +385,6 @@ class array return m_global_size; } - void resize(const size_type size) { resize(size, m_default_value); } - void local_clear() { resize(0); } void local_swap(self_type& other) { diff --git a/include/ygm/container/detail/block_partitioner.hpp b/include/ygm/container/detail/block_partitioner.hpp index 83e88a7b..0a488ea6 100644 --- a/include/ygm/container/detail/block_partitioner.hpp +++ b/include/ygm/container/detail/block_partitioner.hpp @@ -69,7 +69,7 @@ struct block_partitioner { index_type local_index(const index_type &global_index) { index_type to_return = global_index - m_local_start_index; - ASSERT_RELEASE((to_return >= 0) && (to_return <= m_small_block_size)); + ASSERT_RELEASE((to_return >= 0) && (to_return < m_local_size)); return to_return; } From 71bdcad0de4a64ca13136ecbd6e37982edb8f706 Mon Sep 17 00:00:00 2001 From: Trevor Steil Date: Thu, 1 Aug 2024 11:33:52 -0700 Subject: [PATCH 03/40] Getting started documentation and reworks README (#226) --- Readme.md | 133 +++++++++++++++-------------------- docs/rtd/getting_started.rst | 116 ++++++++++++++++++++++++++++++ docs/rtd/index.rst | 3 +- docs/rtd/ygm/comm.rst | 51 +++++++++++++- 4 files changed, 225 insertions(+), 78 deletions(-) create mode 100644 docs/rtd/getting_started.rst diff --git a/Readme.md b/Readme.md index f7c12e9f..8ac66c22 100644 --- a/Readme.md +++ b/Readme.md @@ -1,33 +1,70 @@ -# What is YGM? - -YGM is an asynchronous communication library designed for irregular communication patterns. It is built on a -communicator abstraction, much like MPI, but communication is handled asynchronously and is initiated by senders without -any interaction with receivers. YGM features -* **Message buffering** - Increases application throughput. -* **Fire-and-Forget RPC Semantics** - A sender provides the function and function arguments for execution on a specified - destination rank through an `async` call. This function will complete on the destination rank at an unspecified time - in the future, but YGM does not explicitly make the sender aware of this completion. -* **Storage Containers** - YGM provides a collection of distributed storage containers with asynchronous - interfaces, used for many common distributed memory operations. Containers are designed to partition data, allowing -insertions to occur from any rank. Data is accessed through collective `for_all` operations that execute a user-provided -function on every stored object, or, when a particular piece of data's location is known, `visit`-type operations that -perform a user-provided function only on the desired data. These containers are found -[here](/include/ygm/container/). - -# Getting Started +## What is YGM? + +YGM is an asynchronous communication library written in C++ and designed for high-performance computing (HPC) use cases featuring +irregular communication patterns. YGM includes a collection of +distributed-memory storage containers designed to express common algorithmic and data-munging tasks. These containers +automatically partition data, allowing insertions and, with most containers, processing of individual elements to be +initiated from any runninng YGM process. + +Underlying YGM's containers is a communicator abstraction. This communicator asynchronously sends messages spawned by +senders with receivers needing no knowledge of incoming messages prior to their arrival. YGM communications take the +form of *active messages*; each message contains a function object to execute (often in the form of C++ lambdas), data +and/or pointers to data for this function to execute on, and a destination process for the message to be executed at. + +YGM also includes a set of I/O primitives for parsing collections of input documents in parallel as independent lines of +text and streaming output lines to +large numbers of destination files. Current parsing functionality supports reading input as CSV, ndjson, and +unstructured lines of data. + +## General YGM Operations + +YGM is built on its ability to communicate active messages asynchronously between running processes. This does not +capture every operation that can be useful, for instance collective operations are still widely needed. YGM uses +prefixes on function names to distinguish their behaviors in terms of the processes involved. These prefixes are: + * `async_`: Asynchronous operation initiated on a single process. The execution of the underlying function may + occur on a remote process. + * `local_`: Function performs only local operations on data of the current process. In uses within YGM containers + with partitioning schemes that determine item ownership, care must be taken to ensure the process a `local_` + operation is called from aligns with the item's owner. For instance, calling `ygm::container::map::local_insert` + will store an item on the process where the call is made, but the `ygm::container::map` may not be able to look + up this location if it is on the wrong process. + * No Prefix: Collective operation that must be called from all processes. + +The primary workhorse functions in YGM fall into the two categories of `async_` and `for_all` operations. In an +`async_` operation, a lambda is asynchronously sent to a (potentially) remote process for execution. In many cases +with YGM containers, the lambda being executed is not provided by the user and is instead part of the function itself, +e.g. `async_insert` calls on most containers. A `for_all` operation is a collective operation in which a lambda is +executed locally on every process while iterating over all locally held items of some YGM object. The items iterated +over can be items in a YGM container, items coming from a map, filter, or flatten applied to a container, or all lines +in a collection of files in a YGM I/O parser. + +### Lambda Capture Rules +Certain `async_` and `for_all` operations require users to provide lambdas as part of their executions. The lambdas +that can be accepted by these two classes of functions follow different rules pertaining to the capturing of variables: + * `async_` calls cannot capture (most) variables in lambdas. Variables necessary for lambda execution must be + provided as arguments to the `async_` call. In the event that the data for the lambda resides on the remote + process the lambda will execute on, a `ygm::ygm_ptr` should be passed as an argument to the `async_`. + * `for_all` calls assume lambdas take only the arguments inherently provided by the YGM object being iterated over. + All other necessary variables *must* be captured. The types of arguments provided to the lambda can be identified + by the `for_all_args` type within the YGM object. + +These differences in behavior arise from the distinction that `async_` lambdas may execute on a remote process, while +`for_all` lambdas are guaranteed to execute locally to a process. In the case of `async_` operations, the lambda and +all arguments must be serialized for communication, but C++ does not provide a method for inspection of variables +captured in the closure of a lambda. In the case of `for_all` operations, the execution is equivalent to calling +[`std::for_each`](https://en.cppreference.com/w/cpp/algorithm/for_each) on entire collection of items held locally. ## Requirements -* C++17 - GCC versions 8, 9 and 10 are tested. Your mileage may vary with other compilers. +* C++20 - GCC versions 11 and 12 are tested. Your mileage may vary with other compilers. * [Cereal](https://github.com/USCiLab/cereal) - C++ serialization library * MPI * Optionally, Boost 1.77 to enable Boost.JSON support. - ## Using YGM with CMake YGM is a header-only library that is easy to incorporate into a project through CMake. Adding the following to CMakeLists.txt will install YGM and its dependencies as part of your project: ``` -set(DESIRED_YGM_VERSION 0.4) +set(DESIRED_YGM_VERSION 0.6) find_package(ygm ${DESIRED_YGM_VERSION} CONFIG) if (NOT ygm_FOUND) FetchContent_Declare( @@ -52,62 +89,6 @@ else () endif () ``` -# Anatomy of a YGM Program -Here we will walk through a basic "hello world" YGM program. The [examples directory](/examples/) contains several other -examples, including many using YGM's storage containers. - -To begin, headers for a YGM communicator are needed -``` C++ -#include -``` - -At the beginning of the program, a YGM communicator must be constructed. It will be given `argc` and `argv` like -`MPI_Init`, and it has an optional third argument that specifies the aggregate size (in bytes) allowed for all send -buffers before YGM begins flushing sends. Here, we will make a buffer with 32MB of aggregate send buffer space. -``` C++ -ygm::comm world(&argc, &argv, 32*1024*1024); -``` - -Next, we need a lambda to send through YGM. We'll do a simple hello\_world type of lambda. -``` C++ -auto hello_world_lambda = [](const std::string &name) { - std::cout << "Hello " << name << std::endl; -}; -``` - -Finally, we use this lambda inside of our `async` calls. In this case, we will have rank 0 send a message to rank 1, -telling it to greet the world -``` C++ -if (world.rank0()) { - world.async(1, hello_world_lambda, std::string("world")); -} -``` - -The full, compilable version of this example is found [here](/examples/hello_world.cpp). Running it prints a single -"Hello world". - -# Potential Pitfalls - -## Allowed Lambdas -There are two distinct classes of lambdas that can be given to YGM: *remote lambdas* and *local lambdas*, each of which -has different requirements. - -### Remote Lambdas -A *remote lambda* is any lambda that may potentially be executed on a different rank. These lambdas are identified as -being those given to a `ygm::comm` or any of the storage containers through a function prefixed by `async_`. - -The defining feature of remote lambdas is they **must not** capture any variables; all variables must be provided as -arguments. This limitation is due to the lack of -ability for YGM to inspect and extract these arguments when serializing messages to be sent to other ranks. - -### Local Lambdas -A *local lambda* is any lambda that is guaranteed not to be sent to a remote rank. These lambdas are identified as being -those given to a `for_all` operation on a storage container. - -The defining feature of local lambdas is that all arguments besides what is stored in the container must be captured. -Internally, these lambdas may be given to a [`std::for_each`](https://en.cppreference.com/w/cpp/algorithm/for_each) that -iterates over the container's elements stored locally on each rank. - # License YGM is distributed under the MIT license. diff --git a/docs/rtd/getting_started.rst b/docs/rtd/getting_started.rst new file mode 100644 index 00000000..9b4dcbe6 --- /dev/null +++ b/docs/rtd/getting_started.rst @@ -0,0 +1,116 @@ +Getting Started +*************** + +What is YGM? +============ + +YGM is an asynchronous communication library written in C++ and designed for high-performance computing (HPC) use cases featuring +irregular communication patterns. YGM includes a collection of +distributed-memory storage containers designed to express common algorithmic and data-munging tasks. These containers +automatically partition data, allowing insertions and, with most containers, processing of individual elements to be +initiated from any runninng YGM process. + +Underlying YGM's containers is a communicator abstraction. This communicator asynchronously sends messages spawned by +senders with receivers needing no knowledge of incoming messages prior to their arrival. YGM communications take the +form of *active messages*; each message contains a function object to execute (often in the form of C++ lambdas), data +and/or pointers to data for this function to execute on, and a destination process for the message to be executed at. + +YGM also includes a set of I/O primitives for parsing collections of input documents in parallel as independent lines of +text and streaming output lines to +large numbers of destination files. Current parsing functionality supports reading input as CSV, ndjson, and +unstructured lines of data. + +General YGM Operations +====================== + +YGM is built on its ability to communicate active messages asynchronously between running processes. This does not +capture every operation that can be useful, for instance collective operations are still widely needed. YGM uses +prefixes on function names to distinguish their behaviors in terms of the processes involved. These prefixes are: + * ``async_``: Asynchronous operation initiated on a single process. The execution of the underlying function may + occur on a remote process. + * ``local_``: Function performs only local operations on data of the current process. In uses within YGM containers + with partitioning schemes that determine item ownership, care must be taken to ensure the process a ``local_`` + operation is called from aligns with the item's owner. For instance, calling ``ygm::container::map::local_insert`` + will store an item on the process where the call is made, but the ``ygm::container::map`` may not be able to look + up this location if it is on the wrong process. + * No Prefix: Collective operation that must be called from all processes. + +The primary workhorse functions in YGM fall into the two categories of ``async_`` and ``for_all`` operations. In an +``async_`` operation, a lambda is asynchronously sent to a (potentially) remote process for execution. In many cases +with YGM containers, the lambda being executed is not provided by the user and is instead part of the function itself, +e.g. ``async_insert`` calls on most containers. A ``for_all`` operation is a collective operation in which a lambda is +executed locally on every process while iterating over all locally held items of some YGM object. The items iterated +over can be items in a YGM container, items coming from a map, filter, or flatten applied to a container, or all lines +in a collection of files in a YGM I/O parser. + +Lambda Capture Rules +-------------------- +Certain ``async_`` and ``for_all`` operations require users to provide lambdas as part of their executions. The lambdas +that can be accepted by these two classes of functions follow different rules pertaining to the capturing of variables: + * ``async_`` calls cannot capture (most) variables in lambdas. Variables necessary for lambda execution must be + provided as arguments to the ``async_`` call. In the event that the data for the lambda resides on the remote + process the lambda will execute on, a ``ygm::ygm_ptr`` should be passed as an argument to the ``async_``. + * ``for_all`` calls assume lambdas take only the arguments inherently provided by the YGM object being iterated over. + All other necessary variables *must* be captured. The types of arguments provided to the lambda can be identified + by the ``for_all_args`` type within the YGM object. + +These differences in behavior arise from the distinction that ``async_`` lambdas may execute on a remote process, while +``for_all`` lambdas are guaranteed to execute locally to a process. In the case of ``async_`` operations, the lambda and +all arguments must be serialized for communication, but C++ does not provide a method for inspection of variables +captured in the closure of a lambda. In the case of ``for_all`` operations, the execution is equivalent to calling +`std::for_each `_ on entire collection of items held locally. + +Requirements +============ + +* C++20 - GCC versions 11 and 12 are tested. Your mileage may vary with other compilers. +* `Cereal `_ - C++ serialization library +* MPI +* Optionally, Boost 1.77 to enable Boost.JSON support. + + +Using YGM with CMake +==================== +YGM is a header-only library that is easy to incorporate into a project through CMake. Adding the following to +CMakeLists.txt will install YGM and its dependencies as part of your project: + +.. code-block:: CMake + + set(DESIRED_YGM_VERSION 0.6) + find_package(ygm ${DESIRED_YGM_VERSION} CONFIG) + if (NOT ygm_FOUND) + FetchContent_Declare( + ygm + GIT_REPOSITORY https://github.com/LLNL/ygm + GIT_TAG v${DESIRED_YGM_VERSION} + ) + FetchContent_GetProperties(ygm) + if (ygm_POPULATED) + message(STATUS "Found already populated ygm dependency: " + ${ygm_SOURCE_DIR} + ) + else () + set(JUST_INSTALL_YGM ON) + set(YGM_INSTALL ON) + FetchContent_Populate(ygm) + add_subdirectory(${ygm_SOURCE_DIR} ${ygm_BINARY_DIR}) + message(STATUS "Cloned ygm dependency " ${ygm_SOURCE_DIR}) + endif () + else () + message(STATUS "Found installed ygm dependency " ${ygm_DIR}) + endif () + +License +======= +YGM is distributed under the MIT license. + +All new contributions must be made under the MIT license. + +See [LICENSE-MIT](LICENSE-MIT), [NOTICE](NOTICE), and [COPYRIGHT](COPYRIGHT) for +details. + +SPDX-License-Identifier: MIT + +Release +======= +LLNL-CODE-789122 diff --git a/docs/rtd/index.rst b/docs/rtd/index.rst index a32c2d2b..0f429a5a 100644 --- a/docs/rtd/index.rst +++ b/docs/rtd/index.rst @@ -10,6 +10,7 @@ YGM library documentation :maxdepth: 2 :caption: Contents: + getting_started ygm/comm ygm/container @@ -26,4 +27,4 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` -* :ref:`search` \ No newline at end of file +* :ref:`search` diff --git a/docs/rtd/ygm/comm.rst b/docs/rtd/ygm/comm.rst index 7c4340a7..440073eb 100644 --- a/docs/rtd/ygm/comm.rst +++ b/docs/rtd/ygm/comm.rst @@ -3,10 +3,59 @@ :code:`ygm::comm` class reference. ================================== +Communicator Overview +===================== + The communicator :code:`ygm::comm` is the central object in YGM. The communicator controls an interface to an MPI communicator, and its functionality can be modified by additional optional parameters. +Communicator Features: + * **Message Buffering** - Increases application throughput at the expense of increased message latency. + * **Message Routing** - Extends benefits of message buffering to extremely large HPC allocations. + * **Fire-and-Forget RPC Semantics** - A sender provides the function and function arguments for execution on a specified + destination rank through an `async` call. This function will complete on the destination rank at an unspecified time + in the future, but YGM does not explicitly make the sender aware of this completion. + +Communicator Hello World +======================== + +Here we will walk through a basic "hello world" YGM program. The [examples directory](/examples/) contains several other +examples, including many using YGM's storage containers. + +To begin, headers for a YGM communicator are needed: + +.. code-block:: C++ + + #include + +At the beginning of the program, a YGM communicator must be constructed. It will be given ``argc`` and ``argv`` like +``MPI_Init``. + +.. code-block:: C++ + + ygm::comm world(&argc, &argv); + +Next, we need a lambda to send through YGM. We'll do a simple hello\_world type of lambda. + +.. code-block:: C++ + + auto hello_world_lambda = [](const std::string &name) { + std::cout << "Hello " << name << std::endl; + }; + +Finally, we use this lambda inside of our `async` calls. In this case, we will have rank 0 send a message to rank 1, +telling it to greet the world + +.. code-block:: C++ + + if (world.rank0()) { + world.async(1, hello_world_lambda, std::string("world")); + } + +The full, compilable version of this example is found `here `_. Running it prints a single +"Hello world". + .. doxygenclass:: ygm::comm :members: - :undoc-members: \ No newline at end of file + :undoc-members: From 73df3fc504530b708b6da68d59d59cb5aa838823 Mon Sep 17 00:00:00 2001 From: Trevor Steil Date: Thu, 1 Aug 2024 11:35:31 -0700 Subject: [PATCH 04/40] Makes dereferencing a const ygm_ptr return a const pointer to non-const T (#223) --- include/ygm/detail/ygm_ptr.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/ygm/detail/ygm_ptr.hpp b/include/ygm/detail/ygm_ptr.hpp index 79071837..acef84e7 100644 --- a/include/ygm/detail/ygm_ptr.hpp +++ b/include/ygm/detail/ygm_ptr.hpp @@ -18,10 +18,9 @@ class ygm_ptr { ygm_ptr(){}; T *operator->() { return sptrs[idx]; } - const T *operator->() const { return sptrs[idx]; } + T *const operator->() const { return sptrs[idx]; } - T &operator*() { return *sptrs[idx]; } - const T &operator*() const { return *sptrs[idx]; } + T &operator*() const { return *sptrs[idx]; } /** * @brief Construct a new ygm ptr object From 28677520f7dce45388e22d969eab8e7fd5cf4d8e Mon Sep 17 00:00:00 2001 From: Trevor Steil Date: Thu, 1 Aug 2024 15:05:27 -0700 Subject: [PATCH 05/40] CSV Parser Headers (#214) * Adds ability to handle headers in CSV files. Requires allowing ygm::io::line_parser to read the first line of the first file independently and skip over the first line of files in for_all operations. * Makes ygm::io::detail::csv_line::size() const * Adds ability to check if ygm::io::csv_parser headers contain a particular label * Fixes use of ygm::comm::mpi_bcast() in ygm::io::line_parser::read_first() * Makes skipping first line in line parser controlled by a member variable --- include/ygm/io/csv_parser.hpp | 42 ++++++++++++++++--- include/ygm/io/detail/csv.hpp | 74 +++++++++++++++++++++++++++++++++- include/ygm/io/line_parser.hpp | 29 ++++++++++++- test/CMakeLists.txt | 1 + test/data/csv_headers.csv | 5 +++ test/test_csv_headers.cpp | 35 ++++++++++++++++ 6 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 test/data/csv_headers.csv create mode 100644 test/test_csv_headers.cpp diff --git a/include/ygm/io/csv_parser.hpp b/include/ygm/io/csv_parser.hpp index 8b003dd3..5163ab65 100644 --- a/include/ygm/io/csv_parser.hpp +++ b/include/ygm/io/csv_parser.hpp @@ -6,6 +6,7 @@ #pragma once #include +#include #include #include #include @@ -19,7 +20,8 @@ class csv_parser : public ygm::container::detail::base_iteration>; template - csv_parser(Args&&... args) : m_lp(std::forward(args)...) {} + csv_parser(Args&&... args) + : m_lp(std::forward(args)...), m_has_headers(false) {} /** * @brief Executes a user function for every CSV record in a set of files. @@ -30,17 +32,45 @@ class csv_parser : public ygm::container::detail::base_iteration void for_all(Function fn) { using namespace ygm::io::detail; - m_lp.for_all([fn](const std::string& line) { - auto vfields = parse_csv_line(line); + + std::map* header_map_ptr; + bool skip_first; + auto handle_line_lambda = [fn, this](const std::string& line) { + auto vfields = parse_csv_line(line, m_header_map); // auto stypes = convert_type_string(vfields); // todo, detect if types are inconsistent between records if (vfields.size() > 0) { fn(vfields); } - }); + }; + + m_lp.for_all(handle_line_lambda); + } + + /** + * @brief Read the header of a CSV file + */ + void read_headers() { + using namespace ygm::io::detail; + auto header_line = m_lp.read_first_line(); + m_lp.set_skip_first_line(true); + m_header_map = parse_csv_headers(header_line); + m_has_headers = true; + } + + /** + * @brief Checks for existence of a column label within headers + * + * @param label Header label to search for within headers + */ + bool has_header(const std::string& label) { + return m_has_headers && (m_header_map.find(label) != m_header_map.end()); } private: line_parser m_lp; -}; -} // namespace ygm::io \ No newline at end of file + + std::map m_header_map; + bool m_has_headers; +}; // namespace ygm::io +} // namespace ygm::io diff --git a/include/ygm/io/detail/csv.hpp b/include/ygm/io/detail/csv.hpp index 2e54bbac..3b3fe4fc 100644 --- a/include/ygm/io/detail/csv.hpp +++ b/include/ygm/io/detail/csv.hpp @@ -6,6 +6,7 @@ #pragma once #include +#include #include #include #include @@ -68,12 +69,57 @@ class csv_field { std::string m_f; }; +class csv_line { + public: + using vector_type = std::vector; + using size_type = vector_type::size_type; + using reference = vector_type::reference; + using const_reference = vector_type::const_reference; + using iterator = vector_type::iterator; + using const_iterator = vector_type::const_iterator; + using reverse_iterator = vector_type::reverse_iterator; + using const_reverse_iterator = vector_type::const_reverse_iterator; + + csv_line(const std::map &header_map) + : m_header_map_ref(header_map){}; + + void push_back(const csv_field &f) { m_csv_fields.push_back(f); } + + size_type size() const { return m_csv_fields.size(); } + + reference operator[](size_type n) { return m_csv_fields[n]; } + + const_reference operator[](size_type n) const { return m_csv_fields[n]; } + + const_reference operator[](const std::string &key) const { + return m_csv_fields[m_header_map_ref.at(key)]; + } + + iterator begin() { return m_csv_fields.begin(); } + iterator end() { return m_csv_fields.end(); } + const_iterator begin() const { return m_csv_fields.begin(); } + const_iterator end() const { return m_csv_fields.end(); } + reverse_iterator rbegin() { return m_csv_fields.rbegin(); } + reverse_iterator rend() { return m_csv_fields.rend(); } + const_reverse_iterator rbegin() const { return m_csv_fields.rbegin(); } + const_reverse_iterator rend() const { return m_csv_fields.rend(); } + const_iterator cbegin() const { return m_csv_fields.cbegin(); } + const_iterator cend() const { return m_csv_fields.cend(); } + const_reverse_iterator crbegin() const { return m_csv_fields.crbegin(); } + const_reverse_iterator crend() const { return m_csv_fields.crend(); } + + private: + std::vector m_csv_fields; + const std::map &m_header_map_ref; +}; + std::ostream &operator<<(std::ostream &os, const csv_field &f) { return os << f.as_string(); } -std::vector parse_csv_line(const std::string line) { - std::vector line_fields; +csv_line parse_csv_line(const std::string line, + const std::map &header_map_ref) { + csv_line line_fields(header_map_ref); if (line.empty() || line[0] == '#') { return line_fields; } @@ -96,6 +142,30 @@ std::vector parse_csv_line(const std::string line) { return line_fields; } +std::map parse_csv_headers(const std::string header_line) { + std::map header_map; + + std::stringstream ssline(header_line); + int column_num{0}; + while (ssline >> std::ws) { + std::string header_field; + if (ssline.peek() == '"') { + ssline >> std::quoted(header_field); + if (ssline) { + header_map[header_field] = column_num++; + } + ssline.ignore(256, ','); + } else { + std::getline(ssline, header_field, ','); + if (ssline) { + header_map[header_field] = column_num++; + } + } + } + + return header_map; +} + std::string convert_type_string(const std::vector &line_fields) { std::stringstream ss; for (const auto &f : line_fields) { diff --git a/include/ygm/io/line_parser.hpp b/include/ygm/io/line_parser.hpp index d25178e4..aa9916fd 100644 --- a/include/ygm/io/line_parser.hpp +++ b/include/ygm/io/line_parser.hpp @@ -32,7 +32,9 @@ class line_parser: public ygm::container::detail::base_iteration& stringpaths, bool node_local_filesystem = false, bool recursive = false) - : m_comm(comm), m_node_local_filesystem(node_local_filesystem) { + : m_comm(comm), + m_node_local_filesystem(node_local_filesystem), + m_skip_first_line(false) { if (node_local_filesystem) { ASSERT_RELEASE(false); check_paths(stringpaths, recursive); @@ -124,22 +126,44 @@ class line_parser: public ygm::container::detail::base_iteration 0) { ifs.seekg(bytes_begin); std::getline(ifs, line); + } else { + first_line = true; } // Keep reading until line containing bytes_end is read while (ifs.tellg() <= bytes_end && std::getline(ifs, line)) { - fn(line); + // Skip first line if necessary + if (not first_line || not m_skip_first_line) { + fn(line); + } else { + } // if(ifs.tellg() > bytes_end) break; + first_line = false; } } my_file_paths.clear(); } } + std::string read_first_line() { + std::string line; + if (m_comm.rank0()) { + std::ifstream ifs(m_paths[0]); + std::getline(ifs, line); + } + + line = m_comm.mpi_bcast(line, 0, m_comm.get_mpi_comm()); + + return line; + } + + void set_skip_first_line(bool skip_first) { m_skip_first_line = skip_first; } + private: /** * @brief Check readability of paths and iterates through directories @@ -212,6 +236,7 @@ class line_parser: public ygm::container::detail::base_iteration m_paths; bool m_node_local_filesystem; + bool m_skip_first_line; }; } // namespace ygm::io diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 01c35a9f..71bc7189 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -50,6 +50,7 @@ add_ygm_test(test_disjoint_set) #add_ygm_test(test_container_serialization) add_ygm_test(test_line_parser) add_ygm_test(test_csv_parser) +add_ygm_test(test_csv_headers) add_ygm_test(test_multi_output) add_ygm_test(test_daily_output) add_ygm_test(test_interrupt_mask) diff --git a/test/data/csv_headers.csv b/test/data/csv_headers.csv new file mode 100644 index 00000000..129f06ad --- /dev/null +++ b/test/data/csv_headers.csv @@ -0,0 +1,5 @@ +zero, four, two, six +0, 4, 2, 6 +0, 4, 2, 6 +0, 4, 2, 6 +0, 4, 2, 6 diff --git a/test/test_csv_headers.cpp b/test/test_csv_headers.cpp new file mode 100644 index 00000000..c892574f --- /dev/null +++ b/test/test_csv_headers.cpp @@ -0,0 +1,35 @@ +// Copyright 2019-2021 Lawrence Livermore National Security, LLC and other YGM +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: MIT + +#undef NDEBUG + +#include +#include +#include + +int main(int argc, char** argv) { + ygm::comm world(&argc, &argv); + + ygm::io::csv_parser csvp(world, + std::vector{"data/csv_headers.csv"}); + csvp.read_headers(); + csvp.for_all([&world](const auto& vfields) { + // Test lookups by header names + ASSERT_RELEASE(vfields["zero"].as_integer() == 0); + ASSERT_RELEASE(vfields["two"].as_integer() == 2); + ASSERT_RELEASE(vfields["four"].as_integer() == 4); + ASSERT_RELEASE(vfields["six"].as_integer() == 6); + + // Test lookup by column names agrees with positional lookups + ASSERT_RELEASE(vfields["zero"].as_integer() == vfields[0].as_integer()); + ASSERT_RELEASE(vfields["two"].as_integer() == vfields[2].as_integer()); + ASSERT_RELEASE(vfields["four"].as_integer() == vfields[1].as_integer()); + ASSERT_RELEASE(vfields["six"].as_integer() == vfields[3].as_integer()); + }); + + world.barrier(); + + return 0; +} From 9e5da3d3f061af9a3115395ba0cd20b134ae977d Mon Sep 17 00:00:00 2001 From: Roger Pearce Date: Fri, 2 Aug 2024 12:26:38 -0500 Subject: [PATCH 06/40] Updated container constructors. (#227) --- include/ygm/container/bag.hpp | 21 +++- include/ygm/container/bag_orig.hpp | 72 ----------- include/ygm/container/counting_set.hpp | 44 ++++++- include/ygm/container/map.hpp | 72 +++++++++++ include/ygm/container/old_set.hpp | 164 ------------------------- include/ygm/container/set.hpp | 79 +++++++++++- include/ygm/for_all_adapter.hpp | 50 -------- test/test_set.cpp | 25 +++- 8 files changed, 233 insertions(+), 294 deletions(-) delete mode 100644 include/ygm/container/bag_orig.hpp delete mode 100644 include/ygm/container/old_set.hpp delete mode 100644 include/ygm/for_all_adapter.hpp diff --git a/include/ygm/container/bag.hpp b/include/ygm/container/bag.hpp index b5c1ab62..97ae0d17 100644 --- a/include/ygm/container/bag.hpp +++ b/include/ygm/container/bag.hpp @@ -37,24 +37,39 @@ class bag : public detail::base_async_insert_value, std::tuple>, bag(ygm::comm &comm, std::initializer_list l) : m_comm(comm), pthis(this), partitioner(comm) { - m_comm.cout0("initializer_list assumes all ranks are equal"); pthis.check(m_comm); if (m_comm.rank0()) { for (const Item &i : l) { async_insert(i); } } + m_comm.barrier(); } template bag(ygm::comm &comm, const STLContainer &cont) + requires detail::STLContainer && + std::convertible_to : m_comm(comm), pthis(this), partitioner(comm) { - m_comm.cout0("STLContainer assumes all ranks are different"); pthis.check(m_comm); for (const Item &i : cont) { - async_insert(i); + this->async_insert(i); } + m_comm.barrier(); + } + + template + bag(ygm::comm &comm, const YGMContainer &yc) + requires detail::HasForAll && + detail::SingleItemTuple< + typename YGMContainer::for_all_args> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + yc.for_all([this](const Item &value) { this->async_insert(value); }); + + m_comm.barrier(); } ~bag() { m_comm.barrier(); } diff --git a/include/ygm/container/bag_orig.hpp b/include/ygm/container/bag_orig.hpp deleted file mode 100644 index 2ecd6795..00000000 --- a/include/ygm/container/bag_orig.hpp +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2019-2021 Lawrence Livermore National Security, LLC and other YGM -// Project Developers. See the top-level COPYRIGHT file for details. -// -// SPDX-License-Identifier: MIT - -#pragma once - -#include -#include - -namespace ygm::container { -template > -class bag { - public: - using self_type = bag; - using value_type = Item; - using size_type = size_t; - using ygm_for_all_types = std::tuple; - using container_type = ygm::container::bag_tag; - - bag(ygm::comm &comm); - ~bag(); - - void async_insert(const value_type &item); - void async_insert(const value_type &item, int dest); - void async_insert(const std::vector &items, int dest); - - template - void for_all(Function fn); - - void clear(); - - size_type size(); - size_type local_size(); - - void rebalance(); - - void swap(self_type &s); - - template - void local_shuffle(RandomFunc &r); - void local_shuffle(); - - template - void global_shuffle(RandomFunc &r); - void global_shuffle(); - - template - void local_for_all(Function fn); - - ygm::comm &comm(); - - void serialize(const std::string &fname); - void deserialize(const std::string &fname); - std::vector gather_to_vector(int dest); - std::vector gather_to_vector(); - - private: - std::vector local_pop(int n); - - template - void local_for_all_pair_types(Function fn); - - private: - size_t m_round_robin = 0; - ygm::comm &m_comm; - std::vector m_local_bag; - typename ygm::ygm_ptr pthis; -}; -} // namespace ygm::container - -#include diff --git a/include/ygm/container/counting_set.hpp b/include/ygm/container/counting_set.hpp index cb1e364b..6194a3e1 100644 --- a/include/ygm/container/counting_set.hpp +++ b/include/ygm/container/counting_set.hpp @@ -34,7 +34,7 @@ class counting_set const size_type count_cache_size = 1024 * 1024; counting_set(ygm::comm &comm) - : m_map(comm /*, mapped_type(0)*/), + : m_map(comm ), m_comm(comm), partitioner(m_map.partitioner), pthis(this) { @@ -43,9 +43,47 @@ class counting_set counting_set() = delete; - void async_insert(const key_type &key) { cache_insert(key); } + counting_set(ygm::comm &comm, std::initializer_list l) + : m_map(comm ), + m_comm(comm), + partitioner(m_map.partitioner), + pthis(this) { + pthis.check(m_comm); + if (m_comm.rank0()) { + for (const Key &i : l) { + async_insert(i); + } + } + m_comm.barrier(); +} + + template + counting_set(ygm::comm &comm, const STLContainer &cont) + requires detail::STLContainer && + std::convertible_to + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + for (const Key &i : cont) { + this->async_insert(i); + } + m_comm.barrier(); + } + + template + counting_set(ygm::comm &comm, const YGMContainer &yc) + requires detail::HasForAll && + detail::SingleItemTuple< + typename YGMContainer::for_all_args> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); - // void async_erase(const key_type& key) { cache_erase(key); } + yc.for_all([this](const Key &value) { this->async_insert(value); }); + + m_comm.barrier(); + } + + void async_insert(const key_type &key) { cache_insert(key); } template void local_for_all(Function fn) { diff --git a/include/ygm/container/map.hpp b/include/ygm/container/map.hpp index 4af07fe8..f0630022 100644 --- a/include/ygm/container/map.hpp +++ b/include/ygm/container/map.hpp @@ -52,6 +52,42 @@ class map pthis.check(m_comm); } + map(ygm::comm& comm, std::initializer_list> l) + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + if (m_comm.rank0()) { + for (const std::pair& i : l) { + async_insert(i); + } + } + } + + template + map(ygm::comm &comm, const STLContainer &cont) + requires detail::STLContainer && + std::convertible_to> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + for (const std::pair &i : cont) { + this->async_insert(i); + } + m_comm.barrier(); + } + + template + map(ygm::comm &comm, const YGMContainer &yc) + requires detail::HasForAll && + detail::SingleItemTuple< + typename YGMContainer::for_all_args> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + yc.for_all([this](const std::pair &value) { this->async_insert(value); }); + + m_comm.barrier(); + } + ~map() { m_comm.barrier(); } using detail::base_async_erase_key, @@ -326,6 +362,42 @@ class multimap pthis.check(m_comm); } + multimap(ygm::comm& comm, std::initializer_list> l) + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + if (m_comm.rank0()) { + for (const std::pair& i : l) { + async_insert(i); + } + } + } + + template + multimap(ygm::comm &comm, const STLContainer &cont) + requires detail::STLContainer && + std::convertible_to> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + for (const std::pair &i : cont) { + this->async_insert(i); + } + m_comm.barrier(); + } + + template + multimap(ygm::comm &comm, const YGMContainer &yc) + requires detail::HasForAll && + detail::SingleItemTuple< + typename YGMContainer::for_all_args> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + yc.for_all([this](const std::pair &value) { this->async_insert(value); }); + + m_comm.barrier(); + } + ~multimap() { m_comm.barrier(); } void local_insert(const key_type& key) { diff --git a/include/ygm/container/old_set.hpp b/include/ygm/container/old_set.hpp deleted file mode 100644 index 609e9db8..00000000 --- a/include/ygm/container/old_set.hpp +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2019-2021 Lawrence Livermore National Security, LLC and other YGM -// Project Developers. See the top-level COPYRIGHT file for details. -// -// SPDX-License-Identifier: MIT - -#pragma once - -#include -#include - -namespace ygm::container { - -template , - typename Compare = std::less, - class Alloc = std::allocator> -class multiset { - public: - using self_type = multiset; - using key_type = Key; - using size_type = size_t; - using ygm_for_all_types = std::tuple; - using impl_type = detail::set_impl; - - Partitioner partitioner; - - multiset() = delete; - - multiset(ygm::comm& comm) : m_impl(comm) {} - - void async_insert(const key_type& key) { m_impl.async_insert_multi(key); } - - void async_erase(const key_type& key) { m_impl.async_erase(key); } - - template - void for_all(Function fn) { - m_impl.for_all(fn); - } - - template - void consume_all(Function fn) { - m_impl.consume_all(fn); - } - - void clear() { m_impl.clear(); } - - size_type size() { return m_impl.size(); } - - bool empty() { return m_impl.size() == 0; } - - size_t count(const key_type& key) { return m_impl.count(key); } - - void swap(self_type& s) { return m_impl.swap(s.m_impl); } - - void serialize(const std::string& fname) { m_impl.serialize(fname); } - void deserialize(const std::string& fname) { m_impl.deserialize(fname); } - - typename ygm::ygm_ptr get_ygm_ptr() const { - return m_impl.get_ygm_ptr(); - } - - template - void local_for_all(Function fn) { - m_impl.local_for_all(fn); - } - - int owner(const key_type& key) const { return m_impl.owner(key); } - - ygm::comm& comm() { return m_impl.comm(); } - - private: - impl_type m_impl; -}; - -template , - typename Compare = std::less, - class Alloc = std::allocator> -class set { - public: - using self_type = set; - using key_type = Key; - using size_type = size_t; - using ygm_container_type = ygm::container::set_tag; - using ygm_for_all_types = std::tuple; - using impl_type = detail::set_impl; - - Partitioner partitioner; - - set() = delete; - - set(ygm::comm& comm) : m_impl(comm) {} - - void async_insert(const key_type& key) { m_impl.async_insert_unique(key); } - - void async_erase(const key_type& key) { m_impl.async_erase(key); } - - template - void async_insert_exe_if_missing(const key_type& key, Visitor visitor, - const VisitorArgs&... args) { - m_impl.async_insert_exe_if_missing( - key, visitor, std::forward(args)...); - } - - template - void async_insert_exe_if_contains(const key_type& key, Visitor visitor, - const VisitorArgs&... args) { - m_impl.async_insert_exe_if_contains( - key, visitor, std::forward(args)...); - } - - template - void async_exe_if_missing(const key_type& key, Visitor visitor, - const VisitorArgs&... args) { - m_impl.async_exe_if_missing(key, visitor, - std::forward(args)...); - } - - template - void async_exe_if_contains(const key_type& key, Visitor visitor, - const VisitorArgs&... args) { - m_impl.async_exe_if_contains(key, visitor, - std::forward(args)...); - } - - template - void for_all(Function fn) { - m_impl.for_all(fn); - } - - template - void consume_all(Function fn) { - m_impl.consume_all(fn); - } - - void clear() { m_impl.clear(); } - - size_type size() { return m_impl.size(); } - - bool empty() { return m_impl.size() == 0; } - - size_t count(const key_type& key) { return m_impl.count(key); } - - void swap(self_type& s) { return m_impl.swap(s.m_impl); } - - void serialize(const std::string& fname) { m_impl.serialize(fname); } - void deserialize(const std::string& fname) { m_impl.deserialize(fname); } - - typename ygm::ygm_ptr get_ygm_ptr() const { - return m_impl.get_ygm_ptr(); - } - - template - void local_for_all(Function fn) { - m_impl.local_for_all(fn); - } - - int owner(const key_type& key) const { return m_impl.owner(key); } - - ygm::comm& comm() { return m_impl.comm(); } - - private: - impl_type m_impl; -}; - -} // namespace ygm::container diff --git a/include/ygm/container/set.hpp b/include/ygm/container/set.hpp index 3267a34e..cddf9d0c 100644 --- a/include/ygm/container/set.hpp +++ b/include/ygm/container/set.hpp @@ -15,7 +15,6 @@ #include #include #include -// #include namespace ygm::container { @@ -59,6 +58,46 @@ class multiset pthis.check(m_comm); } + multiset(ygm::comm &comm, std::initializer_list l) + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + if (m_comm.rank0()) { + for (const Value &i : l) { + async_insert(i); + } + } + + m_comm.barrier(); + } + + template + multiset(ygm::comm &comm, const STLContainer &cont) + requires detail::STLContainer && + std::convertible_to + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + for (const Value &i : cont) { + this->async_insert(i); + } + + m_comm.barrier(); + } + + template + multiset(ygm::comm &comm, const YGMContainer &yc) + requires detail::HasForAll && + detail::SingleItemTuple< + typename YGMContainer::for_all_args> //&& + // std::same_as> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + yc.for_all([this](const Value &value) { this->async_insert(value); }); + + m_comm.barrier(); + } + ~multiset() { m_comm.barrier(); } multiset() = delete; @@ -146,6 +185,44 @@ class set pthis.check(m_comm); } + set(ygm::comm &comm, std::initializer_list l) + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + if (m_comm.rank0()) { + for (const Value &i : l) { + this->async_insert(i); + } + } + m_comm.barrier(); + } + + template + set(ygm::comm &comm, const STLContainer &cont) + requires detail::STLContainer && + std::convertible_to + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + for (const Value &i : cont) { + this->async_insert(i); + } + m_comm.barrier(); + } + + template + set(ygm::comm &comm, const YGMContainer &yc) + requires detail::HasForAll && + detail::SingleItemTuple< + typename YGMContainer::for_all_args> //&& + // std::same_as> + : m_comm(comm), pthis(this), partitioner(comm) { + pthis.check(m_comm); + + yc.for_all([this](const Value &value) { this->async_insert(value); }); + + m_comm.barrier(); + } + ~set() { m_comm.barrier(); } set() = delete; diff --git a/include/ygm/for_all_adapter.hpp b/include/ygm/for_all_adapter.hpp deleted file mode 100644 index 0b87c063..00000000 --- a/include/ygm/for_all_adapter.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2019-2021 Lawrence Livermore National Security, LLC and other YGM -// Project Developers. See the top-level COPYRIGHT file for details. -// -// SPDX-License-Identifier: MIT - -#pragma once - -namespace ygm { - -/** - * @brief Consuming for_all adapter. - * - * @tparam Container - */ -template -class for_all_consume_adapter { - public: - for_all_consume_adapter(Container& c) : m_rc(c) {} - - template - void for_all(Function fn) { - m_rc.consume_all(fn); - } - - private: - Container& m_rc; -}; - -/** - * @brief Adapter that iteratively calls consume_all until container is globally - * empty. - * - * @tparam Container - */ -template -class consume_all_iterative_adapter { - public: - consume_all_iterative_adapter(Container& c) : m_rc(c) {} - - template - void consume_all(Function fn) { - while (not m_rc.empty()) { - m_rc.consume_all(fn); - } - } - - private: - Container& m_rc; -}; -} // namespace ygm \ No newline at end of file diff --git a/test/test_set.cpp b/test/test_set.cpp index 9072e86d..dd74446c 100644 --- a/test/test_set.cpp +++ b/test/test_set.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include int main(int argc, char **argv) { ygm::comm world(&argc, &argv); @@ -142,6 +142,29 @@ int main(int argc, char **argv) { } + // Test from bag + { + ygm::container::bag sbag(world, {"one", "two", "three", "one", "two"}); + ASSERT_RELEASE(sbag.size() == 5); + + ygm::container::set sset(world, sbag); + ASSERT_RELEASE(sset.size() == 3); + } + + // Test initializer list + { + ygm::container::set sset(world, {"one", "two", "three", "one", "two"}); + ASSERT_RELEASE(sset.size() == 3); + } + + // Test from STL vector + { + std::vector v({1,2,3,4,5,1,1,1,3}); + ygm::container::set iset(world, v); + ASSERT_RELEASE(iset.size() == 5); + } + + // // Test additional arguments of async_contains // { From 6f0c7876a58d2a1ceb84c9d0a216518e6d06a57c Mon Sep 17 00:00:00 2001 From: Roger Pearce Date: Fri, 2 Aug 2024 12:41:23 -0500 Subject: [PATCH 07/40] Added prefix YGM_ to ASSERT macros. (#228) --- include/ygm/collective.hpp | 20 ++-- include/ygm/container/array.hpp | 10 +- include/ygm/container/bag.hpp | 2 +- include/ygm/container/counting_set.hpp | 6 +- include/ygm/container/detail/bag_orig.ipp | 2 +- .../ygm/container/detail/base_iteration.hpp | 2 +- .../container/detail/block_partitioner.hpp | 10 +- .../container/detail/disjoint_set_impl.hpp | 10 +- include/ygm/container/detail/map_impl.hpp | 4 +- .../ygm/container/detail/reducing_adapter.hpp | 2 +- include/ygm/detail/assert.hpp | 6 +- include/ygm/detail/comm.ipp | 88 ++++++++--------- include/ygm/detail/lambda_map.hpp | 2 +- include/ygm/detail/layout.hpp | 22 ++--- include/ygm/detail/mpi.hpp | 4 +- include/ygm/detail/ygm_cereal_archive.hpp | 4 +- include/ygm/detail/ygm_ptr.hpp | 2 +- include/ygm/io/line_parser.hpp | 6 +- test/test_array.cpp | 90 ++++++++--------- test/test_bag.cpp | 96 +++++++++--------- test/test_barrier.cpp | 2 +- test/test_cereal_archive.cpp | 2 +- test/test_cereal_boost_container.cpp | 2 +- test/test_cereal_boost_json.cpp | 2 +- test/test_collective.cpp | 40 ++++---- test/test_comm.cpp | 28 +++--- test/test_comm_2.cpp | 6 +- test/test_container_serialization.cpp | 96 +++++++++--------- test/test_counting_set.cpp | 90 ++++++++--------- test/test_csv_headers.cpp | 16 +-- test/test_csv_parser.cpp | 4 +- test/test_daily_output.cpp | 6 +- test/test_disjoint_set.cpp | 74 +++++++------- test/test_interrupt_mask.cpp | 4 +- test/test_large_messages.cpp | 2 +- test/test_layout.cpp | 30 +++--- test/test_line_parser.cpp | 10 +- test/test_map.cpp | 86 ++++++++-------- test/test_multi_output.cpp | 6 +- test/test_multimap.cpp | 98 +++++++++---------- test/test_multiset.cpp | 54 +++++----- test/test_ndjson_parser.cpp | 2 +- test/test_parquet_reader.cpp | 10 +- test/test_parquet_reader_json.cpp | 78 +++++++-------- test/test_random.cpp | 8 +- test/test_recursion_large_messages.cpp | 2 +- test/test_recursion_progress.cpp | 2 +- test/test_reduce_by_key.cpp | 14 +-- test/test_reducing_adapter.cpp | 10 +- test/test_set.cpp | 90 ++++++++--------- test/test_tagged_bag.cpp | 10 +- 51 files changed, 636 insertions(+), 636 deletions(-) diff --git a/include/ygm/collective.hpp b/include/ygm/collective.hpp index 89e2fc0e..1251a1a1 100644 --- a/include/ygm/collective.hpp +++ b/include/ygm/collective.hpp @@ -23,7 +23,7 @@ T prefix_sum(const T &value, const comm &c) { T to_return{0}; c.barrier(); MPI_Comm mpi_comm = c.get_mpi_comm(); - ASSERT_MPI(MPI_Exscan(&value, &to_return, 1, detail::mpi_typeof(value), + YGM_ASSERT_MPI(MPI_Exscan(&value, &to_return, 1, detail::mpi_typeof(value), MPI_SUM, mpi_comm)); return to_return; } @@ -42,7 +42,7 @@ T sum(const T &value, const comm &c) { T to_return; c.barrier(); MPI_Comm mpi_comm = c.get_mpi_comm(); - ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(T()), + YGM_ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(T()), MPI_SUM, mpi_comm)); return to_return; } @@ -61,7 +61,7 @@ T min(const T &value, const comm &c) { T to_return; c.barrier(); MPI_Comm mpi_comm = c.get_mpi_comm(); - ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(T()), + YGM_ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(T()), MPI_MIN, mpi_comm)); return to_return; } @@ -80,7 +80,7 @@ T max(const T &value, const comm &c) { T to_return; c.barrier(); MPI_Comm mpi_comm = c.get_mpi_comm(); - ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(T()), + YGM_ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(T()), MPI_MAX, mpi_comm)); return to_return; } @@ -98,7 +98,7 @@ inline bool logical_and(bool value, const comm &c) { bool to_return; c.barrier(); MPI_Comm mpi_comm = c.get_mpi_comm(); - ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(bool()), + YGM_ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(bool()), MPI_LAND, mpi_comm)); return to_return; } @@ -116,7 +116,7 @@ inline bool logical_or(bool value, const comm &c) { bool to_return; c.barrier(); MPI_Comm mpi_comm = c.get_mpi_comm(); - ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(bool()), + YGM_ASSERT_MPI(MPI_Allreduce(&value, &to_return, 1, detail::mpi_typeof(bool()), MPI_LOR, mpi_comm)); return to_return; } @@ -133,7 +133,7 @@ template void bcast(T &to_bcast, int root, const comm &cm) { if constexpr (std::is_trivially_copyable::value && std::is_standard_layout::value) { - ASSERT_MPI( + YGM_ASSERT_MPI( MPI_Bcast(&to_bcast, sizeof(T), MPI_BYTE, root, cm.get_mpi_comm())); } else { std::vector packed; @@ -142,13 +142,13 @@ void bcast(T &to_bcast, int root, const comm &cm) { oarchive(to_bcast); } size_t packed_size = packed.size(); - ASSERT_RELEASE(packed_size < 1024 * 1024 * 1024); - ASSERT_MPI(MPI_Bcast(&packed_size, 1, ygm::detail::mpi_typeof(packed_size), + YGM_ASSERT_RELEASE(packed_size < 1024 * 1024 * 1024); + YGM_ASSERT_MPI(MPI_Bcast(&packed_size, 1, ygm::detail::mpi_typeof(packed_size), root, cm.get_mpi_comm())); if (cm.rank() != root) { packed.resize(packed_size); } - ASSERT_MPI(MPI_Bcast(packed.data(), packed_size, MPI_BYTE, root, + YGM_ASSERT_MPI(MPI_Bcast(packed.data(), packed_size, MPI_BYTE, root, cm.get_mpi_comm())); if (cm.rank() != root) { diff --git a/include/ygm/container/array.hpp b/include/ygm/container/array.hpp index 49b19538..915995a1 100644 --- a/include/ygm/container/array.hpp +++ b/include/ygm/container/array.hpp @@ -98,7 +98,7 @@ class array key_type max_index{0}; for (const auto& [index, value] : l) { - ASSERT_RELEASE(index >= 0); + YGM_ASSERT_RELEASE(index >= 0); max_index = std::max(max_index, index); } @@ -278,7 +278,7 @@ class array void async_binary_op_update_value(const key_type index, const mapped_type& value, const BinaryOp& b) { - ASSERT_RELEASE(index < m_global_size); + YGM_ASSERT_RELEASE(index < m_global_size); auto updater = [](const key_type i, mapped_type& v, const mapped_type& new_value) { BinaryOp* binary_op; @@ -326,7 +326,7 @@ class array template void async_unary_op_update_value(const key_type index, const UnaryOp& u) { - ASSERT_RELEASE(index < m_global_size); + YGM_ASSERT_RELEASE(index < m_global_size); auto updater = [](const key_type i, mapped_type& v) { UnaryOp* u; v = (*u)(v); @@ -446,7 +446,7 @@ class array } m_comm.barrier(); - ASSERT_RELEASE(samples.size() == samples_per_pivot * (m_comm.size() - 1)); + YGM_ASSERT_RELEASE(samples.size() == samples_per_pivot * (m_comm.size() - 1)); std::sort(samples.begin(), samples.end()); for (size_t i = samples_per_pivot - 1; i < samples.size(); i += samples_per_pivot) { @@ -455,7 +455,7 @@ class array samples.clear(); samples.shrink_to_fit(); - ASSERT_RELEASE(pivots.size() == m_comm.size() - 1); + YGM_ASSERT_RELEASE(pivots.size() == m_comm.size() - 1); // // Partition using pivots diff --git a/include/ygm/container/bag.hpp b/include/ygm/container/bag.hpp index 97ae0d17..86511f5b 100644 --- a/include/ygm/container/bag.hpp +++ b/include/ygm/container/bag.hpp @@ -243,7 +243,7 @@ class bag : public detail::base_async_insert_value, std::tuple>, private: std::vector local_pop(int n) { - ASSERT_RELEASE(n <= local_size()); + YGM_ASSERT_RELEASE(n <= local_size()); size_t new_size = local_size() - n; auto pop_start = m_local_bag.begin() + new_size; diff --git a/include/ygm/container/counting_set.hpp b/include/ygm/container/counting_set.hpp index 6194a3e1..e5992b7b 100644 --- a/include/ygm/container/counting_set.hpp +++ b/include/ygm/container/counting_set.hpp @@ -168,12 +168,12 @@ class counting_set m_count_cache[slot].second = 1; } else { // flush slot, fill with key - ASSERT_DEBUG(m_count_cache[slot].second > 0); + YGM_ASSERT_DEBUG(m_count_cache[slot].second > 0); if (m_count_cache[slot].first == key) { m_count_cache[slot].second++; } else { count_cache_flush(slot); - ASSERT_DEBUG(m_count_cache[slot].second == -1); + YGM_ASSERT_DEBUG(m_count_cache[slot].second == -1); m_count_cache[slot].first = key; m_count_cache[slot].second = 1; } @@ -186,7 +186,7 @@ class counting_set void count_cache_flush(size_t slot) { auto key = m_count_cache[slot].first; auto cached_count = m_count_cache[slot].second; - ASSERT_DEBUG(cached_count > 0); + YGM_ASSERT_DEBUG(cached_count > 0); m_map.async_visit( key, [](const key_type &key, size_t &count, int32_t to_add) { diff --git a/include/ygm/container/detail/bag_orig.ipp b/include/ygm/container/detail/bag_orig.ipp index f64f1e9c..b7ff5d25 100644 --- a/include/ygm/container/detail/bag_orig.ipp +++ b/include/ygm/container/detail/bag_orig.ipp @@ -240,7 +240,7 @@ bag::gather_to_vector() { template std::vector::value_type> bag::local_pop( int n) { - ASSERT_RELEASE(n <= local_size()); + YGM_ASSERT_RELEASE(n <= local_size()); size_t new_size = local_size() - n; auto pop_start = m_local_bag.begin() + new_size; diff --git a/include/ygm/container/detail/base_iteration.hpp b/include/ygm/container/detail/base_iteration.hpp index 7f0e7cd7..76456529 100644 --- a/include/ygm/container/detail/base_iteration.hpp +++ b/include/ygm/container/detail/base_iteration.hpp @@ -62,7 +62,7 @@ struct base_iteration { { const derived_type* derived_this = static_cast(this); derived_this->comm().barrier(); - ASSERT_RELEASE(derived_this->local_size() > + YGM_ASSERT_RELEASE(derived_this->local_size() > 0); // empty partition not handled yet using value_type = typename std::tuple_element<0, for_all_args>::type; diff --git a/include/ygm/container/detail/block_partitioner.hpp b/include/ygm/container/detail/block_partitioner.hpp index 0a488ea6..3ff26914 100644 --- a/include/ygm/container/detail/block_partitioner.hpp +++ b/include/ygm/container/detail/block_partitioner.hpp @@ -48,7 +48,7 @@ struct block_partitioner { int to_return; // Owner depends on whether index is before switching to small blocks if (index < (m_partitioned_size % m_comm_size) * m_large_block_size) { - ASSERT_RELEASE(m_large_block_size > 0); + YGM_ASSERT_RELEASE(m_large_block_size > 0); to_return = index / m_large_block_size; } else { if (m_small_block_size == 0) { @@ -56,26 +56,26 @@ struct block_partitioner { << m_partitioned_size << "\t" << m_comm_size << "\t" << index << std::endl; } - ASSERT_RELEASE(m_small_block_size > 0); + YGM_ASSERT_RELEASE(m_small_block_size > 0); to_return = (m_partitioned_size % m_comm_size) + (index - (m_partitioned_size % m_comm_size) * m_large_block_size) / m_small_block_size; } - ASSERT_RELEASE((to_return >= 0) && (to_return < m_comm_size)); + YGM_ASSERT_RELEASE((to_return >= 0) && (to_return < m_comm_size)); return to_return; } index_type local_index(const index_type &global_index) { index_type to_return = global_index - m_local_start_index; - ASSERT_RELEASE((to_return >= 0) && (to_return < m_local_size)); + YGM_ASSERT_RELEASE((to_return >= 0) && (to_return < m_local_size)); return to_return; } index_type global_index(const index_type &local_index) { index_type to_return = m_local_start_index + local_index; - ASSERT_RELEASE(to_return < m_partitioned_size); + YGM_ASSERT_RELEASE(to_return < m_partitioned_size); return to_return; } diff --git a/include/ygm/container/detail/disjoint_set_impl.hpp b/include/ygm/container/detail/disjoint_set_impl.hpp index 0748ced8..f4a79798 100644 --- a/include/ygm/container/detail/disjoint_set_impl.hpp +++ b/include/ygm/container/detail/disjoint_set_impl.hpp @@ -50,7 +50,7 @@ class disjoint_set_impl { private: void increase_rank(const rank_type new_rank) { - ASSERT_RELEASE(m_rank < new_rank); + YGM_ASSERT_RELEASE(m_rank < new_rank); m_rank = new_rank; // Only called on roots @@ -184,12 +184,12 @@ class disjoint_set_impl { const auto &my_parent = item_data.second.get_parent(); const auto my_parent_rank_est = item_data.second.get_parent_rank_estimate(); - ASSERT_RELEASE(my_rank >= merging_rank); + YGM_ASSERT_RELEASE(my_rank >= merging_rank); if (my_rank > merging_rank) { return; } else { - ASSERT_RELEASE(my_rank == merging_rank); + YGM_ASSERT_RELEASE(my_rank == merging_rank); if (my_parent == my_item) { // Merging new item onto root. Need to increase rank. item_data.second.increase_rank(merging_rank + 1); @@ -331,12 +331,12 @@ class disjoint_set_impl { const auto &my_parent = item_data.second.get_parent(); const auto my_parent_rank_est = item_data.second.get_parent_rank_estimate(); - ASSERT_RELEASE(my_rank >= merging_rank); + YGM_ASSERT_RELEASE(my_rank >= merging_rank); if (my_rank > merging_rank) { return; } else { - ASSERT_RELEASE(my_rank == merging_rank); + YGM_ASSERT_RELEASE(my_rank == merging_rank); if (my_parent == my_item) { // Has not found new parent item_data.second.increase_rank(merging_rank + 1); } else { // Tell merging item about new parent diff --git a/include/ygm/container/detail/map_impl.hpp b/include/ygm/container/detail/map_impl.hpp index 40d617a3..7cdd7e2f 100644 --- a/include/ygm/container/detail/map_impl.hpp +++ b/include/ygm/container/detail/map_impl.hpp @@ -90,7 +90,7 @@ class map_impl { if (range.first == range.second) { // check if not in range pmap->m_local_map.insert(std::make_pair(key, pmap->m_default_value)); range = pmap->m_local_map.equal_range(key); - ASSERT_DEBUG(range.first != range.second); + YGM_ASSERT_DEBUG(range.first != range.second); } Visitor *vis = nullptr; pmap->local_visit(key, *vis, args...); @@ -110,7 +110,7 @@ class map_impl { if (range.first == range.second) { // check if not in range pmap->m_local_map.insert(std::make_pair(key, pmap->m_default_value)); range = pmap->m_local_map.equal_range(key); - ASSERT_DEBUG(range.first != range.second); + YGM_ASSERT_DEBUG(range.first != range.second); } ygm::detail::interrupt_mask mask(pmap->m_comm); diff --git a/include/ygm/container/detail/reducing_adapter.hpp b/include/ygm/container/detail/reducing_adapter.hpp index 6ddce0de..322decf3 100644 --- a/include/ygm/container/detail/reducing_adapter.hpp +++ b/include/ygm/container/detail/reducing_adapter.hpp @@ -61,7 +61,7 @@ class reducing_adapter { m_cache[slot].value = m_reducer(m_cache[slot].value, value); } else { cache_flush(slot); - ASSERT_DEBUG(m_cache[slot].occupied == false); + YGM_ASSERT_DEBUG(m_cache[slot].occupied == false); m_cache[slot].key = key; m_cache[slot].value = value; m_cache[slot].occupied = true; diff --git a/include/ygm/detail/assert.hpp b/include/ygm/detail/assert.hpp index 7e286858..7420d9a7 100644 --- a/include/ygm/detail/assert.hpp +++ b/include/ygm/detail/assert.hpp @@ -18,7 +18,7 @@ inline void release_assert_fail(const char *assertion, const char *file, throw std::runtime_error(ss.str()); } -#define ASSERT_MPI(a) \ +#define YGM_ASSERT_MPI(a) \ { \ if (a != MPI_SUCCESS) { \ char *error_string = NULL; \ @@ -32,9 +32,9 @@ inline void release_assert_fail(const char *assertion, const char *file, } \ } -#define ASSERT_DEBUG(expr) assert(expr) +#define YGM_ASSERT_DEBUG(expr) assert(expr) -#define ASSERT_RELEASE(expr) \ +#define YGM_ASSERT_RELEASE(expr) \ (static_cast(expr) \ ? void(0) \ : release_assert_fail(#expr, __FILE__, __LINE__, "")) diff --git a/include/ygm/detail/comm.ipp b/include/ygm/detail/comm.ipp index 7cdd8d63..5888ef7e 100644 --- a/include/ygm/detail/comm.ipp +++ b/include/ygm/detail/comm.ipp @@ -36,7 +36,7 @@ inline comm::comm(MPI_Comm mcomm) : m_layout(mcomm), m_router(m_layout, config.routing) { pimpl_if.reset(); int flag(0); - ASSERT_MPI(MPI_Initialized(&flag)); + YGM_ASSERT_MPI(MPI_Initialized(&flag)); if (!flag) { throw std::runtime_error("YGM::COMM ERROR: MPI not initialized"); } @@ -44,9 +44,9 @@ inline comm::comm(MPI_Comm mcomm) } inline void comm::comm_setup(MPI_Comm c) { - ASSERT_MPI(MPI_Comm_dup(c, &m_comm_async)); - ASSERT_MPI(MPI_Comm_dup(c, &m_comm_barrier)); - ASSERT_MPI(MPI_Comm_dup(c, &m_comm_other)); + YGM_ASSERT_MPI(MPI_Comm_dup(c, &m_comm_async)); + YGM_ASSERT_MPI(MPI_Comm_dup(c, &m_comm_barrier)); + YGM_ASSERT_MPI(MPI_Comm_dup(c, &m_comm_other)); m_vec_send_buffers.resize(m_layout.size()); @@ -112,20 +112,20 @@ inline void comm::stats_print(const std::string &name, std::ostream &os) { inline comm::~comm() { barrier(); - ASSERT_RELEASE(MPI_Barrier(m_comm_async) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Barrier(m_comm_async) == MPI_SUCCESS); - ASSERT_RELEASE(m_send_queue.empty()); - ASSERT_RELEASE(m_send_dest_queue.empty()); - ASSERT_RELEASE(m_send_buffer_bytes == 0); - ASSERT_RELEASE(m_pending_isend_bytes == 0); + YGM_ASSERT_RELEASE(m_send_queue.empty()); + YGM_ASSERT_RELEASE(m_send_dest_queue.empty()); + YGM_ASSERT_RELEASE(m_send_buffer_bytes == 0); + YGM_ASSERT_RELEASE(m_pending_isend_bytes == 0); for (size_t i = 0; i < m_recv_queue.size(); ++i) { - ASSERT_RELEASE(MPI_Cancel(&(m_recv_queue[i].request)) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Cancel(&(m_recv_queue[i].request)) == MPI_SUCCESS); } - ASSERT_RELEASE(MPI_Barrier(m_comm_async) == MPI_SUCCESS); - ASSERT_RELEASE(MPI_Comm_free(&m_comm_async) == MPI_SUCCESS); - ASSERT_RELEASE(MPI_Comm_free(&m_comm_barrier) == MPI_SUCCESS); - ASSERT_RELEASE(MPI_Comm_free(&m_comm_other) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Barrier(m_comm_async) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Comm_free(&m_comm_async) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Comm_free(&m_comm_barrier) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Comm_free(&m_comm_other) == MPI_SUCCESS); pimpl_if.reset(); } @@ -136,7 +136,7 @@ inline void comm::async(int dest, AsyncFunction fn, const SendArgs &...args) { std::is_standard_layout::value, "comm::async() AsyncFunction must be is_trivially_copyable & " "is_standard_layout."); - ASSERT_RELEASE(dest < m_layout.size()); + YGM_ASSERT_RELEASE(dest < m_layout.size()); stats.async(dest); check_if_production_halt_required(); @@ -242,8 +242,8 @@ inline void comm::barrier() { flush_all_local_and_process_incoming(); } } - ASSERT_RELEASE(m_pre_barrier_callbacks.empty()); - ASSERT_RELEASE(m_send_dest_queue.empty()); + YGM_ASSERT_RELEASE(m_pre_barrier_callbacks.empty()); + YGM_ASSERT_RELEASE(m_send_dest_queue.empty()); cf_barrier(); } @@ -254,7 +254,7 @@ inline void comm::barrier() { * called it. See: MPI_Barrier() */ inline void comm::cf_barrier() const { - ASSERT_MPI(MPI_Barrier(m_comm_barrier)); + YGM_ASSERT_MPI(MPI_Barrier(m_comm_barrier)); } template @@ -272,7 +272,7 @@ inline void comm::register_pre_barrier_callback( template inline T comm::all_reduce_sum(const T &t) const { T to_return; - ASSERT_MPI(MPI_Allreduce(&t, &to_return, 1, detail::mpi_typeof(T()), MPI_SUM, + YGM_ASSERT_MPI(MPI_Allreduce(&t, &to_return, 1, detail::mpi_typeof(T()), MPI_SUM, m_comm_other)); return to_return; } @@ -280,7 +280,7 @@ inline T comm::all_reduce_sum(const T &t) const { template inline T comm::all_reduce_min(const T &t) const { T to_return; - ASSERT_MPI(MPI_Allreduce(&t, &to_return, 1, detail::mpi_typeof(T()), MPI_MIN, + YGM_ASSERT_MPI(MPI_Allreduce(&t, &to_return, 1, detail::mpi_typeof(T()), MPI_MIN, m_comm_other)); return to_return; } @@ -288,7 +288,7 @@ inline T comm::all_reduce_min(const T &t) const { template inline T comm::all_reduce_max(const T &t) const { T to_return; - ASSERT_MPI(MPI_Allreduce(&t, &to_return, 1, detail::mpi_typeof(T()), MPI_MAX, + YGM_ASSERT_MPI(MPI_Allreduce(&t, &to_return, 1, detail::mpi_typeof(T()), MPI_MAX, m_comm_other)); return to_return; } @@ -336,20 +336,20 @@ inline void comm::mpi_send(const T &data, int dest, int tag, cereal::YGMOutputArchive oarchive(packed); oarchive(data); size_t packed_size = packed.size(); - ASSERT_RELEASE(packed_size < 1024 * 1024 * 1024); - ASSERT_MPI(MPI_Send(&packed_size, 1, detail::mpi_typeof(packed_size), dest, + YGM_ASSERT_RELEASE(packed_size < 1024 * 1024 * 1024); + YGM_ASSERT_MPI(MPI_Send(&packed_size, 1, detail::mpi_typeof(packed_size), dest, tag, comm)); - ASSERT_MPI(MPI_Send(packed.data(), packed_size, MPI_BYTE, dest, tag, comm)); + YGM_ASSERT_MPI(MPI_Send(packed.data(), packed_size, MPI_BYTE, dest, tag, comm)); } template inline T comm::mpi_recv(int source, int tag, MPI_Comm comm) const { std::vector packed; size_t packed_size{0}; - ASSERT_MPI(MPI_Recv(&packed_size, 1, detail::mpi_typeof(packed_size), source, + YGM_ASSERT_MPI(MPI_Recv(&packed_size, 1, detail::mpi_typeof(packed_size), source, tag, comm, MPI_STATUS_IGNORE)); packed.resize(packed_size); - ASSERT_MPI(MPI_Recv(packed.data(), packed_size, MPI_BYTE, source, tag, comm, + YGM_ASSERT_MPI(MPI_Recv(packed.data(), packed_size, MPI_BYTE, source, tag, comm, MPI_STATUS_IGNORE)); T to_return; @@ -366,13 +366,13 @@ inline T comm::mpi_bcast(const T &to_bcast, int root, MPI_Comm comm) const { oarchive(to_bcast); } size_t packed_size = packed.size(); - ASSERT_RELEASE(packed_size < 1024 * 1024 * 1024); - ASSERT_MPI( + YGM_ASSERT_RELEASE(packed_size < 1024 * 1024 * 1024); + YGM_ASSERT_MPI( MPI_Bcast(&packed_size, 1, detail::mpi_typeof(packed_size), root, comm)); if (rank() != root) { packed.resize(packed_size); } - ASSERT_MPI(MPI_Bcast(packed.data(), packed_size, MPI_BYTE, root, comm)); + YGM_ASSERT_MPI(MPI_Bcast(packed.data(), packed_size, MPI_BYTE, root, comm)); cereal::YGMInputArchive iarchive(packed.data(), packed.size()); T to_return; @@ -467,11 +467,11 @@ inline std::pair comm::barrier_reduce_counts() { uint64_t local_counts[2] = {m_recv_count, m_send_count}; uint64_t global_counts[2] = {0, 0}; - ASSERT_RELEASE(m_pending_isend_bytes == 0); - ASSERT_RELEASE(m_send_buffer_bytes == 0); + YGM_ASSERT_RELEASE(m_pending_isend_bytes == 0); + YGM_ASSERT_RELEASE(m_send_buffer_bytes == 0); MPI_Request req = MPI_REQUEST_NULL; - ASSERT_MPI(MPI_Iallreduce(local_counts, global_counts, 2, MPI_UINT64_T, + YGM_ASSERT_MPI(MPI_Iallreduce(local_counts, global_counts, 2, MPI_UINT64_T, MPI_SUM, m_comm_barrier, &req)); stats.iallreduce(); bool iallreduce_complete(false); @@ -487,7 +487,7 @@ inline std::pair comm::barrier_reduce_counts() { { auto timer = stats.waitsome_iallreduce(); while (outcount == 0) { - ASSERT_MPI( + YGM_ASSERT_MPI( MPI_Testsome(2, twin_req, &outcount, twin_indices, twin_status)); } } @@ -501,7 +501,7 @@ inline std::pair comm::barrier_reduce_counts() { mpi_irecv_request req_buffer = m_recv_queue.front(); m_recv_queue.pop_front(); int buffer_size{0}; - ASSERT_MPI(MPI_Get_count(&twin_status[i], MPI_BYTE, &buffer_size)); + YGM_ASSERT_MPI(MPI_Get_count(&twin_status[i], MPI_BYTE, &buffer_size)); stats.irecv(twin_status[i].MPI_SOURCE, buffer_size); handle_next_receive(req_buffer.buffer, buffer_size); flush_all_local_and_process_incoming(); @@ -528,11 +528,11 @@ inline void comm::flush_send_buffer(int dest) { } request.buffer->swap(m_vec_send_buffers[dest]); if (config.freq_issend > 0 && counter++ % config.freq_issend == 0) { - ASSERT_MPI(MPI_Issend(request.buffer->data(), request.buffer->size(), + YGM_ASSERT_MPI(MPI_Issend(request.buffer->data(), request.buffer->size(), MPI_BYTE, dest, 0, m_comm_async, &(request.request))); } else { - ASSERT_MPI(MPI_Isend(request.buffer->data(), request.buffer->size(), + YGM_ASSERT_MPI(MPI_Isend(request.buffer->data(), request.buffer->size(), MPI_BYTE, dest, 0, m_comm_async, &(request.request))); } @@ -623,7 +623,7 @@ inline void comm::flush_all_local_and_process_incoming() { */ inline void comm::flush_to_capacity() { while (m_send_buffer_bytes > config.buffer_size) { - ASSERT_DEBUG(!m_send_dest_queue.empty()); + YGM_ASSERT_DEBUG(!m_send_dest_queue.empty()); int dest = m_send_dest_queue.front(); m_send_dest_queue.pop_front(); flush_send_buffer(dest); @@ -635,7 +635,7 @@ inline void comm::post_new_irecv(std::shared_ptr &recv_buffer) { recv_req.buffer = recv_buffer; //::madvise(recv_req.buffer.get(), config.irecv_size, MADV_DONTNEED); - ASSERT_MPI(MPI_Irecv(recv_req.buffer.get(), config.irecv_size, MPI_BYTE, + YGM_ASSERT_MPI(MPI_Irecv(recv_req.buffer.get(), config.irecv_size, MPI_BYTE, MPI_ANY_SOURCE, MPI_ANY_TAG, m_comm_async, &(recv_req.request))); m_recv_queue.push_back(recv_req); @@ -917,7 +917,7 @@ inline void comm::handle_next_receive(std::shared_ptr buffer, * @return True if receive queue was non-empty, else false */ inline bool comm::process_receive_queue() { - ASSERT_RELEASE(!m_in_process_receive_queue); + YGM_ASSERT_RELEASE(!m_in_process_receive_queue); m_in_process_receive_queue = true; bool received_to_return = false; @@ -939,7 +939,7 @@ inline bool comm::process_receive_queue() { { auto timer = stats.waitsome_isend_irecv(); while (outcount == 0) { - ASSERT_MPI( + YGM_ASSERT_MPI( MPI_Testsome(2, twin_req, &outcount, twin_indices, twin_status)); } } @@ -954,7 +954,7 @@ inline bool comm::process_receive_queue() { mpi_irecv_request req_buffer = m_recv_queue.front(); m_recv_queue.pop_front(); int buffer_size{0}; - ASSERT_MPI(MPI_Get_count(&twin_status[i], MPI_BYTE, &buffer_size)); + YGM_ASSERT_MPI(MPI_Get_count(&twin_status[i], MPI_BYTE, &buffer_size)); stats.irecv(twin_status[i].MPI_SOURCE, buffer_size); handle_next_receive(req_buffer.buffer, buffer_size); } @@ -962,7 +962,7 @@ inline bool comm::process_receive_queue() { } else { if (!m_send_queue.empty()) { int flag(0); - ASSERT_MPI( + YGM_ASSERT_MPI( MPI_Test(&(m_send_queue.front().request), &flag, MPI_STATUS_IGNORE)); stats.isend_test(); if (flag) { @@ -986,14 +986,14 @@ inline bool comm::local_process_incoming() { while (true) { int flag(0); MPI_Status status; - ASSERT_MPI(MPI_Test(&(m_recv_queue.front().request), &flag, &status)); + YGM_ASSERT_MPI(MPI_Test(&(m_recv_queue.front().request), &flag, &status)); stats.irecv_test(); if (flag) { received_to_return = true; mpi_irecv_request req_buffer = m_recv_queue.front(); m_recv_queue.pop_front(); int buffer_size{0}; - ASSERT_MPI(MPI_Get_count(&status, MPI_BYTE, &buffer_size)); + YGM_ASSERT_MPI(MPI_Get_count(&status, MPI_BYTE, &buffer_size)); stats.irecv(status.MPI_SOURCE, buffer_size); handle_next_receive(req_buffer.buffer, buffer_size); } else { diff --git a/include/ygm/detail/lambda_map.hpp b/include/ygm/detail/lambda_map.hpp index 9b9bd978..b5315ead 100644 --- a/include/ygm/detail/lambda_map.hpp +++ b/include/ygm/detail/lambda_map.hpp @@ -34,7 +34,7 @@ class lambda_map { private: template static FuncId record() { - ASSERT_RELEASE(s_map.size() < std::numeric_limits::max()); + YGM_ASSERT_RELEASE(s_map.size() < std::numeric_limits::max()); FuncId to_return = s_map.size(); LambdaType *lp; // scary, but by definition can't capture s_map.push_back(*lp); diff --git a/include/ygm/detail/layout.hpp b/include/ygm/detail/layout.hpp index 20ffcda4..c5037991 100644 --- a/include/ygm/detail/layout.hpp +++ b/include/ygm/detail/layout.hpp @@ -32,31 +32,31 @@ class layout { public: layout(MPI_Comm comm) { // global ranks - ASSERT_MPI(MPI_Comm_size(comm, &m_comm_size)); - ASSERT_MPI(MPI_Comm_rank(comm, &m_comm_rank)); + YGM_ASSERT_MPI(MPI_Comm_size(comm, &m_comm_size)); + YGM_ASSERT_MPI(MPI_Comm_rank(comm, &m_comm_rank)); // local ranks MPI_Comm comm_local; - ASSERT_MPI(MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, m_comm_rank, + YGM_ASSERT_MPI(MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, m_comm_rank, MPI_INFO_NULL, &comm_local)); - ASSERT_MPI(MPI_Comm_size(comm_local, &m_local_size)); - ASSERT_MPI(MPI_Comm_rank(comm_local, &m_local_id)); + YGM_ASSERT_MPI(MPI_Comm_size(comm_local, &m_local_size)); + YGM_ASSERT_MPI(MPI_Comm_rank(comm_local, &m_local_id)); _mpi_allgather(m_comm_rank, m_local_ranks, m_local_size, comm_local); // node ranks MPI_Comm comm_node; - ASSERT_MPI(MPI_Comm_split(comm, m_local_id, m_comm_rank, &comm_node)); - ASSERT_MPI(MPI_Comm_size(comm_node, &m_node_size)); - ASSERT_MPI(MPI_Comm_rank(comm_node, &m_node_id)); + YGM_ASSERT_MPI(MPI_Comm_split(comm, m_local_id, m_comm_rank, &comm_node)); + YGM_ASSERT_MPI(MPI_Comm_size(comm_node, &m_node_size)); + YGM_ASSERT_MPI(MPI_Comm_rank(comm_node, &m_node_id)); _mpi_allgather(m_comm_rank, m_strided_ranks, m_node_size, comm_node); _mpi_allgather(m_local_id, m_rank_to_local, m_comm_size, comm); _mpi_allgather(m_node_id, m_rank_to_node, m_comm_size, comm); - ASSERT_RELEASE(MPI_Comm_free(&comm_local) == MPI_SUCCESS); - ASSERT_RELEASE(MPI_Comm_free(&comm_node) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Comm_free(&comm_local) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Comm_free(&comm_node) == MPI_SUCCESS); } layout(const layout &rhs) @@ -158,7 +158,7 @@ class layout { template void _mpi_allgather(T &_t, std::vector &out_vec, int size, MPI_Comm comm) { out_vec.resize(size); - ASSERT_MPI(MPI_Allgather(&_t, sizeof(_t), MPI_BYTE, &(out_vec[0]), + YGM_ASSERT_MPI(MPI_Allgather(&_t, sizeof(_t), MPI_BYTE, &(out_vec[0]), sizeof(_t), MPI_BYTE, comm)); } diff --git a/include/ygm/detail/mpi.hpp b/include/ygm/detail/mpi.hpp index e026aad7..b0c5bb9f 100644 --- a/include/ygm/detail/mpi.hpp +++ b/include/ygm/detail/mpi.hpp @@ -13,10 +13,10 @@ namespace ygm::detail { class mpi_init_finalize { public: mpi_init_finalize(int *argc, char ***argv) { - ASSERT_MPI(MPI_Init(argc, argv)); + YGM_ASSERT_MPI(MPI_Init(argc, argv)); } ~mpi_init_finalize() { - ASSERT_RELEASE(MPI_Barrier(MPI_COMM_WORLD) == MPI_SUCCESS); + YGM_ASSERT_RELEASE(MPI_Barrier(MPI_COMM_WORLD) == MPI_SUCCESS); if (MPI_Finalize() != MPI_SUCCESS) { std::cerr << "ERROR: MPI_Finilize() != MPI_SUCCESS" << std::endl; exit(-1); diff --git a/include/ygm/detail/ygm_cereal_archive.hpp b/include/ygm/detail/ygm_cereal_archive.hpp index 935af119..dbbce99e 100644 --- a/include/ygm/detail/ygm_cereal_archive.hpp +++ b/include/ygm/detail/ygm_cereal_archive.hpp @@ -87,7 +87,7 @@ class YGMInputArchive //! Reads size bytes of data from the input stream void loadBinary(void *const data, std::streamsize size) { - ASSERT_DEBUG(m_position + size <= m_capacity); + YGM_ASSERT_DEBUG(m_position + size <= m_capacity); std::memcpy(data, m_pdata + m_position, size); m_position += size; @@ -98,7 +98,7 @@ class YGMInputArchive } bool empty() const { - ASSERT_DEBUG(!(m_position > m_capacity)); + YGM_ASSERT_DEBUG(!(m_position > m_capacity)); return m_position == m_capacity; } diff --git a/include/ygm/detail/ygm_ptr.hpp b/include/ygm/detail/ygm_ptr.hpp index acef84e7..c9e946a4 100644 --- a/include/ygm/detail/ygm_ptr.hpp +++ b/include/ygm/detail/ygm_ptr.hpp @@ -44,7 +44,7 @@ class ygm_ptr { template void check(Comm &c) const { - ASSERT_RELEASE(idx == c.all_reduce_min(idx)); + YGM_ASSERT_RELEASE(idx == c.all_reduce_min(idx)); } template diff --git a/include/ygm/io/line_parser.hpp b/include/ygm/io/line_parser.hpp index aa9916fd..f8f7a143 100644 --- a/include/ygm/io/line_parser.hpp +++ b/include/ygm/io/line_parser.hpp @@ -36,7 +36,7 @@ class line_parser: public ygm::container::detail::base_iteration void for_all(Function fn) { if (m_node_local_filesystem) { - ASSERT_RELEASE(false); + YGM_ASSERT_RELEASE(false); if (m_paths.empty()) return; } else { static std::vector> my_file_paths; @@ -123,7 +123,7 @@ class line_parser: public ygm::container::detail::base_iteration(fname); size_t bytes_end = std::get<2>(fname); - ASSERT_RELEASE(ifs.good()); + YGM_ASSERT_RELEASE(ifs.good()); ifs.imbue(std::locale::classic()); std::string line; bool first_line = false; diff --git a/test/test_array.cpp b/test/test_array.cpp index 9dc5e093..e69df9d4 100644 --- a/test/test_array.cpp +++ b/test/test_array.cpp @@ -44,7 +44,7 @@ int main(int argc, char **argv) { } arr.for_all([](const auto index, const auto value) { - ASSERT_RELEASE(index == value); + YGM_ASSERT_RELEASE(index == value); }); } @@ -67,7 +67,7 @@ int main(int argc, char **argv) { } arr.for_all([&world](const auto index, const auto value) { - ASSERT_RELEASE(value == index + 2 * world.size()); + YGM_ASSERT_RELEASE(value == index + 2 * world.size()); }); } @@ -105,7 +105,7 @@ int main(int argc, char **argv) { cumulative_xor = 0; break; } - ASSERT_RELEASE(value == index ^ cumulative_xor); + YGM_ASSERT_RELEASE(value == index ^ cumulative_xor); }); } @@ -128,7 +128,7 @@ int main(int argc, char **argv) { } arr.for_all([&world](const auto index, const auto value) { - ASSERT_RELEASE(value == index + world.size()); + YGM_ASSERT_RELEASE(value == index + world.size()); }); } @@ -148,7 +148,7 @@ int main(int argc, char **argv) { for (int i = 0; i < size; ++i) { arr.async_visit(i, [](const auto index, const auto value) { - ASSERT_RELEASE(value == index); + YGM_ASSERT_RELEASE(value == index); }); } } @@ -169,7 +169,7 @@ int main(int argc, char **argv) { for (int i = 0; i < size; ++i) { arr.async_visit(i, [](auto ptr, const auto index, const auto value) { - ASSERT_RELEASE(value == index); + YGM_ASSERT_RELEASE(value == index); }); } } @@ -193,7 +193,7 @@ int main(int argc, char **argv) { } arr.for_all([&world](const auto value) { - ASSERT_RELEASE(value == world.size() + 1); + YGM_ASSERT_RELEASE(value == world.size() + 1); }); } @@ -209,7 +209,7 @@ int main(int argc, char **argv) { } arr.for_all([](const auto index, const auto value) { - ASSERT_RELEASE(index == value); + YGM_ASSERT_RELEASE(index == value); }); } @@ -233,7 +233,7 @@ int main(int argc, char **argv) { arr.async_visit( index, [](const auto &index, const auto &my_value, const auto &other_value) { - ASSERT_RELEASE(my_value == other_value); + YGM_ASSERT_RELEASE(my_value == other_value); }, value); }); @@ -242,7 +242,7 @@ int main(int argc, char **argv) { arr_copy.async_visit( index, [](const auto &index, const auto &my_value, const auto &other_value) { - ASSERT_RELEASE(my_value == other_value); + YGM_ASSERT_RELEASE(my_value == other_value); }, value); }); @@ -263,24 +263,24 @@ int main(int argc, char **argv) { world.barrier(); - ASSERT_RELEASE(arr.size() == large_size); + YGM_ASSERT_RELEASE(arr.size() == large_size); arr.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 2 * index); + YGM_ASSERT_RELEASE(value == 2 * index); }); arr.resize(small_size); - ASSERT_RELEASE(arr.size() == small_size); + YGM_ASSERT_RELEASE(arr.size() == small_size); arr.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 2 * index); + YGM_ASSERT_RELEASE(value == 2 * index); }); arr.resize(large_size); - ASSERT_RELEASE(arr.size() == large_size); + YGM_ASSERT_RELEASE(arr.size() == large_size); arr.for_all([&small_size](const auto &index, const auto &value) { if (index < small_size) { - ASSERT_RELEASE(value == 2 * index); + YGM_ASSERT_RELEASE(value == 2 * index); } }); } @@ -299,11 +299,11 @@ int main(int argc, char **argv) { world.barrier(); - ASSERT_RELEASE(arr.size() == initial_size); + YGM_ASSERT_RELEASE(arr.size() == initial_size); arr.clear(); - ASSERT_RELEASE(arr.size() == 0); + YGM_ASSERT_RELEASE(arr.size() == 0); } // Test swap @@ -325,28 +325,28 @@ int main(int argc, char **argv) { world.barrier(); - ASSERT_RELEASE(arr1.size() == size1); - ASSERT_RELEASE(arr2.size() == size2); + YGM_ASSERT_RELEASE(arr1.size() == size1); + YGM_ASSERT_RELEASE(arr2.size() == size2); arr1.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 2 * index); + YGM_ASSERT_RELEASE(value == 2 * index); }); arr2.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 3 * index + 1); + YGM_ASSERT_RELEASE(value == 3 * index + 1); }); arr1.swap(arr2); - ASSERT_RELEASE(arr1.size() == size2); - ASSERT_RELEASE(arr2.size() == size1); + YGM_ASSERT_RELEASE(arr1.size() == size2); + YGM_ASSERT_RELEASE(arr2.size() == size1); arr1.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 3 * index + 1); + YGM_ASSERT_RELEASE(value == 3 * index + 1); }); arr2.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 2 * index); + YGM_ASSERT_RELEASE(value == 2 * index); }); } @@ -369,9 +369,9 @@ int main(int argc, char **argv) { arr.for_all([&default_value](const auto &index, const auto &value) { if (index % 2 == 0) { - ASSERT_RELEASE(value == 2 * index); + YGM_ASSERT_RELEASE(value == 2 * index); } else { - ASSERT_RELEASE(value == default_value); + YGM_ASSERT_RELEASE(value == default_value); } }); } @@ -381,7 +381,7 @@ int main(int argc, char **argv) { ygm::container::array arr(world, {1, 3, 5, 7, 9, 11}); arr.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 2 * index + 1); + YGM_ASSERT_RELEASE(value == 2 * index + 1); }); } @@ -394,9 +394,9 @@ int main(int argc, char **argv) { arr.for_all([](const auto &index, const auto &value) { if (index % 2 == 1) { - ASSERT_RELEASE(value == 2 * index); + YGM_ASSERT_RELEASE(value == 2 * index); } else { - ASSERT_RELEASE(value == 0); + YGM_ASSERT_RELEASE(value == 0); } }); } @@ -416,7 +416,7 @@ int main(int argc, char **argv) { ygm::container::array arr(world, b); arr.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == 1); + YGM_ASSERT_RELEASE(value == 1); }); } @@ -434,12 +434,12 @@ int main(int argc, char **argv) { ygm::container::array arr(world, b); - ASSERT_RELEASE(arr.size() == 2 * bag_size - 1); + YGM_ASSERT_RELEASE(arr.size() == 2 * bag_size - 1); arr.for_all([](const auto &index, const auto &value) { if (index % 2 == 0) { - ASSERT_RELEASE(value == index / 2); + YGM_ASSERT_RELEASE(value == index / 2); } else { - ASSERT_RELEASE(value == 0); + YGM_ASSERT_RELEASE(value == 0); } }); } @@ -458,12 +458,12 @@ int main(int argc, char **argv) { ygm::container::array arr(world, m); - ASSERT_RELEASE(arr.size() == 2 * bag_size - 1); + YGM_ASSERT_RELEASE(arr.size() == 2 * bag_size - 1); arr.for_all([](const auto &index, const auto &value) { if (index % 2 == 0) { - ASSERT_RELEASE(value == index / 2); + YGM_ASSERT_RELEASE(value == index / 2); } else { - ASSERT_RELEASE(value == 0); + YGM_ASSERT_RELEASE(value == 0); } }); } @@ -478,9 +478,9 @@ int main(int argc, char **argv) { ygm::container::array arr(world, local_vec); - ASSERT_RELEASE(arr.size() == world.size() * (world.size() + 1) / 2); + YGM_ASSERT_RELEASE(arr.size() == world.size() * (world.size() + 1) / 2); arr.for_all([](const auto &index, const auto &value) { - ASSERT_RELEASE(value == index); + YGM_ASSERT_RELEASE(value == index); }); } @@ -496,9 +496,9 @@ int main(int argc, char **argv) { ygm::container::array arr(world, local_vec); - ASSERT_RELEASE(arr.size() == world.size() * local_size); + YGM_ASSERT_RELEASE(arr.size() == world.size() * local_size); arr.for_all([&world](const auto &index, const auto &value) { - ASSERT_RELEASE(value == float(index % world.size())); + YGM_ASSERT_RELEASE(value == float(index % world.size())); }); } @@ -513,9 +513,9 @@ int main(int argc, char **argv) { ygm::container::array arr(world, local_map); - ASSERT_RELEASE(arr.size() == world.size() * local_size); + YGM_ASSERT_RELEASE(arr.size() == world.size() * local_size); arr.for_all([&world](const auto &index, const auto &value) { - ASSERT_RELEASE(value == float(index % world.size())); + YGM_ASSERT_RELEASE(value == float(index % world.size())); }); } @@ -543,7 +543,7 @@ int main(int argc, char **argv) { arr.sort(); arr.for_all([](const auto index, const auto &value) { - ASSERT_RELEASE(index == value); + YGM_ASSERT_RELEASE(index == value); }); } diff --git a/test/test_bag.cpp b/test/test_bag.cpp index f1b13099..04362208 100644 --- a/test/test_bag.cpp +++ b/test/test_bag.cpp @@ -37,10 +37,10 @@ int main(int argc, char** argv) { bbag.async_insert("apple"); bbag.async_insert("red"); } - ASSERT_RELEASE(bbag.count("dog") == 1); - ASSERT_RELEASE(bbag.count("apple") == 1); - ASSERT_RELEASE(bbag.count("red") == 1); - ASSERT_RELEASE(bbag.size() == 3); + YGM_ASSERT_RELEASE(bbag.count("dog") == 1); + YGM_ASSERT_RELEASE(bbag.count("apple") == 1); + YGM_ASSERT_RELEASE(bbag.count("red") == 1); + YGM_ASSERT_RELEASE(bbag.size() == 3); } // @@ -53,18 +53,18 @@ int main(int argc, char** argv) { // bbag.async_insert("red"); // } // world.barrier(); - // ASSERT_RELEASE(bbag.size() == 3); + // YGM_ASSERT_RELEASE(bbag.size() == 3); // ygm::container::bag bbag2(bbag); - // ASSERT_RELEASE(bbag.size() == 3); - // ASSERT_RELEASE(bbag2.size() == 3); + // YGM_ASSERT_RELEASE(bbag.size() == 3); + // YGM_ASSERT_RELEASE(bbag2.size() == 3); // if (world.rank0()) { // bbag2.async_insert("car"); // } // world.barrier(); - // ASSERT_RELEASE(bbag.size() == 3); - // ASSERT_RELEASE(bbag2.size() == 4); + // YGM_ASSERT_RELEASE(bbag.size() == 3); + // YGM_ASSERT_RELEASE(bbag2.size() == 4); } // @@ -77,18 +77,18 @@ int main(int argc, char** argv) { bbag.async_insert("red"); } world.barrier(); - ASSERT_RELEASE(bbag.size() == 3); + YGM_ASSERT_RELEASE(bbag.size() == 3); ygm::container::bag bbag2(std::move(bbag)); - ASSERT_RELEASE(bbag.size() == 0); - ASSERT_RELEASE(bbag2.size() == 3); + YGM_ASSERT_RELEASE(bbag.size() == 0); + YGM_ASSERT_RELEASE(bbag2.size() == 3); if (world.rank0()) { bbag2.async_insert("car"); } world.barrier(); - ASSERT_RELEASE(bbag.size() == 0); - ASSERT_RELEASE(bbag2.size() == 4); + YGM_ASSERT_RELEASE(bbag.size() == 0); + YGM_ASSERT_RELEASE(bbag2.size() == 4); } // Testing = operator @@ -100,23 +100,23 @@ int main(int argc, char** argv) { bbag.async_insert("red"); } world.barrier(); - ASSERT_RELEASE(bbag.size() == 3); + YGM_ASSERT_RELEASE(bbag.size() == 3); // ygm::container::bag bbag2 = bbag; - // ASSERT_RELEASE(bbag.size() == 3); - // ASSERT_RELEASE(bbag2.size() == 3); + // YGM_ASSERT_RELEASE(bbag.size() == 3); + // YGM_ASSERT_RELEASE(bbag2.size() == 3); // if (world.rank0()) { // bbag2.async_insert("car"); // } // world.barrier(); - // ASSERT_RELEASE(bbag.size() == 3); - // ASSERT_RELEASE(bbag2.size() == 4); + // YGM_ASSERT_RELEASE(bbag.size() == 3); + // YGM_ASSERT_RELEASE(bbag2.size() == 4); ygm::container::bag bbag3 = std::move(bbag); - ASSERT_RELEASE(bbag.size() == 0); - ASSERT_RELEASE(bbag3.size() == 3); + YGM_ASSERT_RELEASE(bbag.size() == 0); + YGM_ASSERT_RELEASE(bbag3.size() == 3); } @@ -128,23 +128,23 @@ int main(int argc, char** argv) { bbag.async_insert("dog"); bbag.async_insert("apple"); bbag.async_insert("red"); - ASSERT_RELEASE(bbag.size() == 3 * (size_t)world.size()); - ASSERT_RELEASE(bbag.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(bbag.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(bbag.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(bbag.size() == 3 * (size_t)world.size()); + YGM_ASSERT_RELEASE(bbag.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(bbag.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(bbag.count("red") == (size_t)world.size()); { std::vector all_data; bbag.gather(all_data, 0); if (world.rank0()) { - ASSERT_RELEASE(all_data.size() == 3 * (size_t)world.size()); + YGM_ASSERT_RELEASE(all_data.size() == 3 * (size_t)world.size()); } } { std::set all_data; bbag.gather(all_data, 0); if (world.rank0()) { - ASSERT_RELEASE(all_data.size() == 3); + YGM_ASSERT_RELEASE(all_data.size() == 3); } } } @@ -156,7 +156,7 @@ int main(int argc, char** argv) { bbag.async_insert(1); bbag.async_insert(2); bbag.async_insert(3); - ASSERT_RELEASE(bbag.reduce(std::plus()) == 6 * world.size()); + YGM_ASSERT_RELEASE(bbag.reduce(std::plus()) == 6 * world.size()); } // @@ -181,7 +181,7 @@ int main(int argc, char** argv) { bbag.local_shuffle(); bbag.global_shuffle(); - ASSERT_RELEASE(bbag.size() == num_of_items); + YGM_ASSERT_RELEASE(bbag.size() == num_of_items); std::vector bag_content; bbag.gather(bag_content, 0); @@ -189,7 +189,7 @@ int main(int argc, char** argv) { for (int i = 0; i < num_of_items; i++) { if (std::find(bag_content.begin(), bag_content.end(), i) == bag_content.end()) { - ASSERT_RELEASE(false); + YGM_ASSERT_RELEASE(false); } } } @@ -208,7 +208,7 @@ int main(int argc, char** argv) { bbag.for_all([&count](std::string& mstr) { ++count; }); int global_count = world.all_reduce_sum(count); world.barrier(); - ASSERT_RELEASE(global_count == 3); + YGM_ASSERT_RELEASE(global_count == 3); } // @@ -225,7 +225,7 @@ int main(int argc, char** argv) { [&count](std::pair& mstr) { count += mstr.second; }); int global_count = world.all_reduce_sum(count); world.barrier(); - ASSERT_RELEASE(global_count == 6); + YGM_ASSERT_RELEASE(global_count == 6); } // // @@ -242,7 +242,7 @@ int main(int argc, char** argv) { // [&count](std::string& first, int& second) { count += second; }); // int global_count = world.all_reduce_sum(count); // world.barrier(); - // ASSERT_RELEASE(global_count == 6); + // YGM_ASSERT_RELEASE(global_count == 6); // } // @@ -252,7 +252,7 @@ int main(int argc, char** argv) { bbag.async_insert("begin", 0); bbag.async_insert("end", world.size() - 1); bbag.rebalance(); - ASSERT_RELEASE(bbag.local_size() == 2); + YGM_ASSERT_RELEASE(bbag.local_size() == 2); } // @@ -271,9 +271,9 @@ int main(int argc, char** argv) { bbag.size() / world.size() + (bbag.size() % world.size() > 0); if (world.rank() < remainder) { - ASSERT_RELEASE(bbag.local_size() == large_block_size); + YGM_ASSERT_RELEASE(bbag.local_size() == large_block_size); } else { - ASSERT_RELEASE(bbag.local_size() == small_block_size); + YGM_ASSERT_RELEASE(bbag.local_size() == small_block_size); } } @@ -294,10 +294,10 @@ int main(int argc, char** argv) { std::set value_set; bbag.gather(value_set, 0); if (world.rank0()) { - ASSERT_RELEASE(value_set.size() == 200); - ASSERT_RELEASE(*std::min_element(value_set.begin(), value_set.end()) == + YGM_ASSERT_RELEASE(value_set.size() == 200); + YGM_ASSERT_RELEASE(*std::min_element(value_set.begin(), value_set.end()) == 0); - ASSERT_RELEASE(*std::max_element(value_set.begin(), value_set.end()) == + YGM_ASSERT_RELEASE(*std::max_element(value_set.begin(), value_set.end()) == 199); } } @@ -313,19 +313,19 @@ int main(int argc, char** argv) { bbag2.async_insert("apple"); bbag2.async_insert("red"); } - ASSERT_RELEASE(bbag2.size() == 3); + YGM_ASSERT_RELEASE(bbag2.size() == 3); bbag2.swap(bbag); - ASSERT_RELEASE(bbag2.size() == 0); + YGM_ASSERT_RELEASE(bbag2.size() == 0); } - ASSERT_RELEASE(bbag.size() == 3); - ASSERT_RELEASE(bbag.count("dog") == 1); - ASSERT_RELEASE(bbag.count("apple") == 1); - ASSERT_RELEASE(bbag.count("red") == 1); + YGM_ASSERT_RELEASE(bbag.size() == 3); + YGM_ASSERT_RELEASE(bbag.count("dog") == 1); + YGM_ASSERT_RELEASE(bbag.count("apple") == 1); + YGM_ASSERT_RELEASE(bbag.count("red") == 1); if (world.rank0()) { bbag.async_insert("car"); } - ASSERT_RELEASE(bbag.size() == 4); - ASSERT_RELEASE(bbag.count("car") == 1); + YGM_ASSERT_RELEASE(bbag.size() == 4); + YGM_ASSERT_RELEASE(bbag.count("car") == 1); } // @@ -346,7 +346,7 @@ int main(int argc, char** argv) { world.barrier(); for (int bag_index = 0; bag_index < num_bags; ++bag_index) { - ASSERT_RELEASE(vec_bags[bag_index].size() == world.size() * 2); + YGM_ASSERT_RELEASE(vec_bags[bag_index].size() == world.size() * 2); } } diff --git a/test/test_barrier.cpp b/test/test_barrier.cpp index 62fc1cae..00a86884 100644 --- a/test/test_barrier.cpp +++ b/test/test_barrier.cpp @@ -16,7 +16,7 @@ int main(int argc, char **argv) { static int round = 0; for (int i = 0; i < num_rounds; ++i) { world.async_bcast( - [](int curr_round) { ASSERT_RELEASE(curr_round == round); }, round); + [](int curr_round) { YGM_ASSERT_RELEASE(curr_round == round); }, round); world.barrier(); diff --git a/test/test_cereal_archive.cpp b/test/test_cereal_archive.cpp index 84c70959..dd91d8f0 100644 --- a/test/test_cereal_archive.cpp +++ b/test/test_cereal_archive.cpp @@ -33,7 +33,7 @@ int main() { // std::cout << tmp << std::endl; out_sentences.push_back(tmp); } - ASSERT_RELEASE(vec_sentences == out_sentences); + YGM_ASSERT_RELEASE(vec_sentences == out_sentences); } return 0; diff --git a/test/test_cereal_boost_container.cpp b/test/test_cereal_boost_container.cpp index 7435b17f..1d41d747 100644 --- a/test/test_cereal_boost_container.cpp +++ b/test/test_cereal_boost_container.cpp @@ -29,7 +29,7 @@ int main() { boost::container::vector load_value; archive(load_value); - ASSERT_RELEASE(original_value == load_value); + YGM_ASSERT_RELEASE(original_value == load_value); } return 0; diff --git a/test/test_cereal_boost_json.cpp b/test/test_cereal_boost_json.cpp index 17376e08..db54c399 100644 --- a/test/test_cereal_boost_json.cpp +++ b/test/test_cereal_boost_json.cpp @@ -44,7 +44,7 @@ int main() { const bj::value original_value = bj::parse(json_string); // std::cout << original_value << std::endl; // std::cout << load_value << std::endl; - ASSERT_RELEASE(original_value == load_value); + YGM_ASSERT_RELEASE(original_value == load_value); } return 0; diff --git a/test/test_collective.cpp b/test/test_collective.cpp index 306beb24..10e22b86 100644 --- a/test/test_collective.cpp +++ b/test/test_collective.cpp @@ -11,26 +11,26 @@ int main(int argc, char** argv) { ygm::comm world(&argc, &argv); - ASSERT_RELEASE(ygm::sum(size_t(1), world) == world.size()); - ASSERT_RELEASE(ygm::sum(double(1), world) == double(world.size())); - ASSERT_RELEASE(ygm::sum(float(1), world) == float(world.size())); + YGM_ASSERT_RELEASE(ygm::sum(size_t(1), world) == world.size()); + YGM_ASSERT_RELEASE(ygm::sum(double(1), world) == double(world.size())); + YGM_ASSERT_RELEASE(ygm::sum(float(1), world) == float(world.size())); - ASSERT_RELEASE(ygm::min(world.rank(), world) == 0); - ASSERT_RELEASE(ygm::min(double(world.rank()), world) == double(0)); - ASSERT_RELEASE(ygm::min(float(world.rank()), world) == float(0)); + YGM_ASSERT_RELEASE(ygm::min(world.rank(), world) == 0); + YGM_ASSERT_RELEASE(ygm::min(double(world.rank()), world) == double(0)); + YGM_ASSERT_RELEASE(ygm::min(float(world.rank()), world) == float(0)); - ASSERT_RELEASE(ygm::max(world.rank(), world) == world.size() - 1); + YGM_ASSERT_RELEASE(ygm::max(world.rank(), world) == world.size() - 1); - ASSERT_RELEASE(ygm::prefix_sum(1, world) == world.rank()); + YGM_ASSERT_RELEASE(ygm::prefix_sum(1, world) == world.rank()); - ASSERT_RELEASE(ygm::logical_and(true, world) == true); - ASSERT_RELEASE(ygm::logical_and(false, world) == false); - ASSERT_RELEASE(ygm::logical_or(true, world) == true); - ASSERT_RELEASE(ygm::logical_or(false, world) == false); + YGM_ASSERT_RELEASE(ygm::logical_and(true, world) == true); + YGM_ASSERT_RELEASE(ygm::logical_and(false, world) == false); + YGM_ASSERT_RELEASE(ygm::logical_or(true, world) == true); + YGM_ASSERT_RELEASE(ygm::logical_or(false, world) == false); if (world.size() > 1) { - ASSERT_RELEASE(ygm::logical_and(world.rank() % 2 == 0, world) == 0); - ASSERT_RELEASE(ygm::logical_or(world.rank() % 2 == 0, world) == 1); + YGM_ASSERT_RELEASE(ygm::logical_and(world.rank() % 2 == 0, world) == 0); + YGM_ASSERT_RELEASE(ygm::logical_or(world.rank() % 2 == 0, world) == 1); } { @@ -40,7 +40,7 @@ int main(int argc, char** argv) { value = 3.14; } ygm::bcast(value, root, world); - ASSERT_RELEASE(value == 3.14); + YGM_ASSERT_RELEASE(value == 3.14); } { @@ -51,23 +51,23 @@ int main(int argc, char** argv) { value = 42; } ygm::bcast(value, root, world); - ASSERT_RELEASE(value == 42); + YGM_ASSERT_RELEASE(value == 42); } } - ASSERT_RELEASE(is_same(42, world)); + YGM_ASSERT_RELEASE(is_same(42, world)); std::set string_set; if (world.rank() == 0) { string_set.insert("Howdy"); string_set.insert("Aggs"); } - ASSERT_RELEASE(not is_same(string_set, world)); + YGM_ASSERT_RELEASE(not is_same(string_set, world)); string_set.insert("Howdy"); string_set.insert("Aggs"); - ASSERT_RELEASE(is_same(string_set, world)); + YGM_ASSERT_RELEASE(is_same(string_set, world)); - ASSERT_RELEASE(not is_same(world.rank(), world)); + YGM_ASSERT_RELEASE(not is_same(world.rank(), world)); return 0; } \ No newline at end of file diff --git a/test/test_comm.cpp b/test/test_comm.cpp index 697f2070..ee879d7e 100644 --- a/test/test_comm.cpp +++ b/test/test_comm.cpp @@ -8,7 +8,7 @@ #include int main(int argc, char** argv) { - ASSERT_MPI(MPI_Init(nullptr, nullptr)); + YGM_ASSERT_MPI(MPI_Init(nullptr, nullptr)); std::vector routing_schemes{"NONE", "NR", "NLNR"}; for (const auto& routing_scheme : routing_schemes) { @@ -28,7 +28,7 @@ int main(int argc, char** argv) { } } world.barrier(); - ASSERT_RELEASE(counter == 1); + YGM_ASSERT_RELEASE(counter == 1); } // @@ -41,7 +41,7 @@ int main(int argc, char** argv) { dest, [](auto pcounter) { (*pcounter)++; }, pcounter); } world.barrier(); - ASSERT_RELEASE(counter == (size_t)world.size()); + YGM_ASSERT_RELEASE(counter == (size_t)world.size()); } // @@ -54,7 +54,7 @@ int main(int argc, char** argv) { } world.barrier(); - ASSERT_RELEASE(counter == 1); + YGM_ASSERT_RELEASE(counter == 1); } { @@ -66,7 +66,7 @@ int main(int argc, char** argv) { } world.barrier(); - ASSERT_RELEASE(counter == num_bcasts * world.size()); + YGM_ASSERT_RELEASE(counter == num_bcasts * world.size()); } // @@ -85,9 +85,9 @@ int main(int argc, char** argv) { world.barrier(); if (world.rank() % 2) { - ASSERT_RELEASE(counter == 0); + YGM_ASSERT_RELEASE(counter == 0); } else { - ASSERT_RELEASE(counter == 1); + YGM_ASSERT_RELEASE(counter == 1); } } @@ -95,13 +95,13 @@ int main(int argc, char** argv) { // Test reductions { auto max = world.all_reduce_max(size_t(world.rank())); - ASSERT_RELEASE(max == (size_t)world.size() - 1); + YGM_ASSERT_RELEASE(max == (size_t)world.size() - 1); auto min = world.all_reduce_min(size_t(world.rank())); - ASSERT_RELEASE(min == 0); + YGM_ASSERT_RELEASE(min == 0); auto sum = world.all_reduce_sum(size_t(world.rank())); - ASSERT_RELEASE(sum == + YGM_ASSERT_RELEASE(sum == (((size_t)world.size() - 1) * (size_t)world.size()) / 2); size_t id = world.rank(); @@ -112,7 +112,7 @@ int main(int argc, char** argv) { return b; } }); - ASSERT_RELEASE(red == 0); + YGM_ASSERT_RELEASE(red == 0); auto red2 = world.all_reduce(id, [](size_t a, size_t b) { if (a > b) { return a; @@ -120,7 +120,7 @@ int main(int argc, char** argv) { return b; } }); - ASSERT_RELEASE(red2 == (size_t)world.size() - 1); + YGM_ASSERT_RELEASE(red2 == (size_t)world.size() - 1); } // @@ -131,10 +131,10 @@ int main(int argc, char** argv) { world.async_bcast([]() { done = true; }); world.local_wait_until([]() { return done; }); world.barrier(); - ASSERT_RELEASE(done); + YGM_ASSERT_RELEASE(done); } } - ASSERT_MPI(MPI_Finalize()); + YGM_ASSERT_MPI(MPI_Finalize()); return 0; } diff --git a/test/test_comm_2.cpp b/test/test_comm_2.cpp index bd697c3a..91a2df5c 100644 --- a/test/test_comm_2.cpp +++ b/test/test_comm_2.cpp @@ -9,11 +9,11 @@ int main(int argc, char** argv) { int provided; - ASSERT_MPI(MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided)); - ASSERT_RELEASE(MPI_THREAD_MULTIPLE == provided); + YGM_ASSERT_MPI(MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided)); + YGM_ASSERT_RELEASE(MPI_THREAD_MULTIPLE == provided); for (size_t i = 0; i < 1000; ++i) { ygm::comm world(MPI_COMM_WORLD); } - ASSERT_MPI(MPI_Finalize()); + YGM_ASSERT_MPI(MPI_Finalize()); return 0; } \ No newline at end of file diff --git a/test/test_container_serialization.cpp b/test/test_container_serialization.cpp index 066391f7..7167b38c 100644 --- a/test/test_container_serialization.cpp +++ b/test/test_container_serialization.cpp @@ -23,7 +23,7 @@ int main(int argc, char** argv) { my_bag.async_insert(5); my_bag.async_insert(8); } - ASSERT_RELEASE(my_bag.size() == 4); + YGM_ASSERT_RELEASE(my_bag.size() == 4); my_bag.serialize("serialization_test.bag"); } @@ -33,7 +33,7 @@ int main(int argc, char** argv) { ygm::container::bag reloaded_bag(world); reloaded_bag.deserialize("serialization_test.bag"); - ASSERT_RELEASE(reloaded_bag.size() == 4); + YGM_ASSERT_RELEASE(reloaded_bag.size() == 4); } } @@ -47,10 +47,10 @@ if (world.rank0()) { my_set.async_insert(3); my_set.async_insert(3); } -ASSERT_RELEASE(my_set.count(0) == 1); -ASSERT_RELEASE(my_set.count(2) == 1); -ASSERT_RELEASE(my_set.count(3) == 1); -ASSERT_RELEASE(my_set.size() == 3); +YGM_ASSERT_RELEASE(my_set.count(0) == 1); +YGM_ASSERT_RELEASE(my_set.count(2) == 1); +YGM_ASSERT_RELEASE(my_set.count(3) == 1); +YGM_ASSERT_RELEASE(my_set.size() == 3); my_set.serialize("serialization_test.set"); } @@ -60,13 +60,13 @@ my_set.serialize("serialization_test.set"); ygm::container::set reloaded_set(world); reloaded_set.deserialize("serialization_test.set"); - ASSERT_RELEASE(reloaded_set.count(0) == 1); - ASSERT_RELEASE(reloaded_set.count(2) == 1); - ASSERT_RELEASE(reloaded_set.count(3) == 1); - ASSERT_RELEASE(reloaded_set.size() == 3); + YGM_ASSERT_RELEASE(reloaded_set.count(0) == 1); + YGM_ASSERT_RELEASE(reloaded_set.count(2) == 1); + YGM_ASSERT_RELEASE(reloaded_set.count(3) == 1); + YGM_ASSERT_RELEASE(reloaded_set.size() == 3); reloaded_set.async_insert(4); - ASSERT_RELEASE(reloaded_set.size() == 4); + YGM_ASSERT_RELEASE(reloaded_set.size() == 4); } } @@ -80,10 +80,10 @@ my_set.serialize("serialization_test.set"); // my_mset.async_insert(3); // my_mset.async_insert(3); // } -// ASSERT_RELEASE(my_mset.count(0) == 1); -// ASSERT_RELEASE(my_mset.count(2) == 1); -// ASSERT_RELEASE(my_mset.count(3) == 2); -// ASSERT_RELEASE(my_mset.size() == 4); +// YGM_ASSERT_RELEASE(my_mset.count(0) == 1); +// YGM_ASSERT_RELEASE(my_mset.count(2) == 1); +// YGM_ASSERT_RELEASE(my_mset.count(3) == 2); +// YGM_ASSERT_RELEASE(my_mset.size() == 4); // my_mset.serialize("serialization_test.set"); // } @@ -93,13 +93,13 @@ my_set.serialize("serialization_test.set"); // ygm::container::set reloaded_mset(world); // reloaded_mset.deserialize("serialization_test.set"); -// ASSERT_RELEASE(reloaded_mset.count(0) == 1); -// ASSERT_RELEASE(reloaded_mset.count(2) == 1); -// ASSERT_RELEASE(reloaded_mset.count(3) == 2); -// ASSERT_RELEASE(reloaded_mset.size() == 4); +// YGM_ASSERT_RELEASE(reloaded_mset.count(0) == 1); +// YGM_ASSERT_RELEASE(reloaded_mset.count(2) == 1); +// YGM_ASSERT_RELEASE(reloaded_mset.count(3) == 2); +// YGM_ASSERT_RELEASE(reloaded_mset.size() == 4); // reloaded_mset.async_insert(4); -// ASSERT_RELEASE(reloaded_mset.size() == 5); +// YGM_ASSERT_RELEASE(reloaded_mset.size() == 5); // } // } @@ -112,9 +112,9 @@ smap.async_insert("dog", "cat"); smap.async_insert("apple", "orange"); smap.async_insert("red", "green"); -ASSERT_RELEASE(smap.count("dog") == 1); -ASSERT_RELEASE(smap.count("apple") == 1); -ASSERT_RELEASE(smap.count("red") == 1); +YGM_ASSERT_RELEASE(smap.count("dog") == 1); +YGM_ASSERT_RELEASE(smap.count("apple") == 1); +YGM_ASSERT_RELEASE(smap.count("red") == 1); smap.serialize("serialization_test.map"); } @@ -124,9 +124,9 @@ smap.serialize("serialization_test.map"); ygm::container::map reloaded_map(world); reloaded_map.deserialize("serialization_test.map"); - ASSERT_RELEASE(reloaded_map.count("dog") == 1); - ASSERT_RELEASE(reloaded_map.count("apple") == 1); - ASSERT_RELEASE(reloaded_map.count("red") == 1); + YGM_ASSERT_RELEASE(reloaded_map.count("dog") == 1); + YGM_ASSERT_RELEASE(reloaded_map.count("apple") == 1); + YGM_ASSERT_RELEASE(reloaded_map.count("red") == 1); } } @@ -139,9 +139,9 @@ smap.async_insert("dog", "cat"); smap.async_insert("apple", "orange"); smap.async_insert("red", "green"); -ASSERT_RELEASE(smap.count("dog") == (size_t)world.size()); -ASSERT_RELEASE(smap.count("apple") == (size_t)world.size()); -ASSERT_RELEASE(smap.count("red") == (size_t)world.size()); +YGM_ASSERT_RELEASE(smap.count("dog") == (size_t)world.size()); +YGM_ASSERT_RELEASE(smap.count("apple") == (size_t)world.size()); +YGM_ASSERT_RELEASE(smap.count("red") == (size_t)world.size()); smap.serialize("serialization_test.mmap"); } @@ -151,9 +151,9 @@ smap.serialize("serialization_test.mmap"); ygm::container::multimap reloaded_mmap(world); reloaded_mmap.deserialize("serialization_test.mmap"); - ASSERT_RELEASE(reloaded_mmap.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(reloaded_mmap.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(reloaded_mmap.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(reloaded_mmap.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(reloaded_mmap.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(reloaded_mmap.count("red") == (size_t)world.size()); } } @@ -168,17 +168,17 @@ smap.serialize("serialization_test.mmap"); cset.async_insert("apple"); cset.async_insert("red"); - ASSERT_RELEASE(cset.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(cset.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); - ASSERT_RELEASE(cset.size() == 3); + YGM_ASSERT_RELEASE(cset.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.size() == 3); auto count_map = cset.all_gather({"dog", "cat", "apple"}); - ASSERT_RELEASE(count_map["dog"] == (size_t) world.size()); - ASSERT_RELEASE(count_map["apple"] == (size_t) world.size()); - ASSERT_RELEASE(cset.count("cat") == 0); + YGM_ASSERT_RELEASE(count_map["dog"] == (size_t) world.size()); + YGM_ASSERT_RELEASE(count_map["apple"] == (size_t) world.size()); + YGM_ASSERT_RELEASE(cset.count("cat") == 0); - ASSERT_RELEASE(cset.count_all() == 3 * (size_t) world.size()); + YGM_ASSERT_RELEASE(cset.count_all() == 3 * (size_t) world.size()); cset.serialize("serialization_test.cset"); } @@ -188,17 +188,17 @@ smap.serialize("serialization_test.mmap"); ygm::container::counting_set reloaded_cset(world); reloaded_cset.deserialize("serialization_test.cset"); - ASSERT_RELEASE(reloaded_cset.count("dog") == (size_t) world.size()); - ASSERT_RELEASE(reloaded_cset.count("apple") == (size_t) world.size()); - ASSERT_RELEASE(reloaded_cset.count("red") == (size_t) world.size()); - ASSERT_RELEASE(reloaded_cset.size() == 3); + YGM_ASSERT_RELEASE(reloaded_cset.count("dog") == (size_t) world.size()); + YGM_ASSERT_RELEASE(reloaded_cset.count("apple") == (size_t) world.size()); + YGM_ASSERT_RELEASE(reloaded_cset.count("red") == (size_t) world.size()); + YGM_ASSERT_RELEASE(reloaded_cset.size() == 3); auto count_map = reloaded_cset.all_gather({"dog", "cat", "apple"}); - ASSERT_RELEASE(count_map["dog"] == (size_t) world.size()); - ASSERT_RELEASE(count_map["apple"] == (size_t) world.size()); - ASSERT_RELEASE(reloaded_cset.count("cat") == 0); + YGM_ASSERT_RELEASE(count_map["dog"] == (size_t) world.size()); + YGM_ASSERT_RELEASE(count_map["apple"] == (size_t) world.size()); + YGM_ASSERT_RELEASE(reloaded_cset.count("cat") == 0); - ASSERT_RELEASE(reloaded_cset.count_all() == 3 * (size_t) world.size()); + YGM_ASSERT_RELEASE(reloaded_cset.count_all() == 3 * (size_t) world.size()); } } return 0; diff --git a/test/test_counting_set.cpp b/test/test_counting_set.cpp index ac6a48ab..19cf6cd1 100644 --- a/test/test_counting_set.cpp +++ b/test/test_counting_set.cpp @@ -35,15 +35,15 @@ int main(int argc, char **argv) { cset.async_insert("red"); } - ASSERT_RELEASE(cset.count("dog") == 1); - ASSERT_RELEASE(cset.count("apple") == 1); - ASSERT_RELEASE(cset.count("red") == 1); - ASSERT_RELEASE(cset.size() == 3); + YGM_ASSERT_RELEASE(cset.count("dog") == 1); + YGM_ASSERT_RELEASE(cset.count("apple") == 1); + YGM_ASSERT_RELEASE(cset.count("red") == 1); + YGM_ASSERT_RELEASE(cset.size() == 3); auto count_map = cset.key_gather({"dog", "cat", "apple"}); - ASSERT_RELEASE(count_map["dog"] == 1); - ASSERT_RELEASE(count_map["apple"] == 1); - ASSERT_RELEASE(count_map.count("cat") == 0); + YGM_ASSERT_RELEASE(count_map["dog"] == 1); + YGM_ASSERT_RELEASE(count_map["apple"] == 1); + YGM_ASSERT_RELEASE(count_map.count("cat") == 0); } // @@ -55,17 +55,17 @@ int main(int argc, char **argv) { cset.async_insert("apple"); cset.async_insert("red"); - ASSERT_RELEASE(cset.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(cset.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); - ASSERT_RELEASE(cset.size() == 3); + YGM_ASSERT_RELEASE(cset.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.size() == 3); auto count_map = cset.key_gather({"dog", "cat", "apple"}); - ASSERT_RELEASE(count_map["dog"] == (size_t)world.size()); - ASSERT_RELEASE(count_map["apple"] == (size_t)world.size()); - ASSERT_RELEASE(cset.count("cat") == 0); + YGM_ASSERT_RELEASE(count_map["dog"] == (size_t)world.size()); + YGM_ASSERT_RELEASE(count_map["apple"] == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count("cat") == 0); - ASSERT_RELEASE(cset.count_all() == 3 * (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count_all() == 3 * (size_t)world.size()); } // @@ -80,17 +80,17 @@ int main(int argc, char **argv) { cset_ptr->async_insert("apple"); cset.async_insert("red"); - ASSERT_RELEASE(cset_ptr->count("dog") == (size_t)world.size()); - ASSERT_RELEASE(cset_ptr->count("apple") == (size_t)world.size()); - ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); - ASSERT_RELEASE(cset.size() == 3); + YGM_ASSERT_RELEASE(cset_ptr->count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset_ptr->count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.size() == 3); auto count_map = cset.key_gather({"dog", "cat", "apple"}); - ASSERT_RELEASE(count_map["dog"] == (size_t)world.size()); - ASSERT_RELEASE(count_map["apple"] == (size_t)world.size()); - ASSERT_RELEASE(cset.count("cat") == 0); + YGM_ASSERT_RELEASE(count_map["dog"] == (size_t)world.size()); + YGM_ASSERT_RELEASE(count_map["apple"] == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count("cat") == 0); - ASSERT_RELEASE(cset.count_all() == 3 * (size_t)world.size()); + YGM_ASSERT_RELEASE(cset.count_all() == 3 * (size_t)world.size()); } // @@ -103,16 +103,16 @@ int main(int argc, char **argv) { cset.async_insert("red"); } - ASSERT_RELEASE(cset.count("dog") == 1); - ASSERT_RELEASE(cset.count("apple") == 1); - ASSERT_RELEASE(cset.count("red") == 1); - ASSERT_RELEASE(cset.size() == 3); + YGM_ASSERT_RELEASE(cset.count("dog") == 1); + YGM_ASSERT_RELEASE(cset.count("apple") == 1); + YGM_ASSERT_RELEASE(cset.count("red") == 1); + YGM_ASSERT_RELEASE(cset.size() == 3); cset.clear(); - ASSERT_RELEASE(cset.size() == 0); - ASSERT_RELEASE(cset.count("dog") == 0); - ASSERT_RELEASE(cset.count("apple") == 0); - ASSERT_RELEASE(cset.count("red") == 0); + YGM_ASSERT_RELEASE(cset.size() == 0); + YGM_ASSERT_RELEASE(cset.count("dog") == 0); + YGM_ASSERT_RELEASE(cset.count("apple") == 0); + YGM_ASSERT_RELEASE(cset.count("red") == 0); } // // @@ -130,10 +130,10 @@ int main(int argc, char **argv) { // auto topk = cset.topk( // 2, [](const auto &a, const auto &b) { return a.second > b.second; }); - // ASSERT_RELEASE(topk[0].first == "dog"); - // ASSERT_RELEASE(topk[0].second == 3 * world.size()); - // ASSERT_RELEASE(topk[1].first == "cat"); - // ASSERT_RELEASE(topk[1].second == 2 * world.size()); + // YGM_ASSERT_RELEASE(topk[0].first == "dog"); + // YGM_ASSERT_RELEASE(topk[0].second == 3 * world.size()); + // YGM_ASSERT_RELEASE(topk[1].first == "cat"); + // YGM_ASSERT_RELEASE(topk[1].second == 2 * world.size()); // } // @@ -149,11 +149,11 @@ int main(int argc, char **argv) { cset1.async_insert("cat"); cset1.async_insert("bird"); - ASSERT_RELEASE(cset1.count("dog") == (size_t)world.size() * 3); - ASSERT_RELEASE(cset1.count("cat") == (size_t)world.size() * 2); - ASSERT_RELEASE(cset1.count("bird") == (size_t)world.size()); - ASSERT_RELEASE(cset1.count("red") == 0); - ASSERT_RELEASE(cset1.size() == 3); + YGM_ASSERT_RELEASE(cset1.count("dog") == (size_t)world.size() * 3); + YGM_ASSERT_RELEASE(cset1.count("cat") == (size_t)world.size() * 2); + YGM_ASSERT_RELEASE(cset1.count("bird") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset1.count("red") == 0); + YGM_ASSERT_RELEASE(cset1.size() == 3); cset1.for_all([&cset2](const auto &key, const auto &value) { for (int i = 0; i < value; i++) { @@ -161,11 +161,11 @@ int main(int argc, char **argv) { } }); - ASSERT_RELEASE(cset2.count("dog") == (size_t)world.size() * 3); - ASSERT_RELEASE(cset2.count("cat") == (size_t)world.size() * 2); - ASSERT_RELEASE(cset2.count("bird") == (size_t)world.size()); - ASSERT_RELEASE(cset2.count("red") == 0); - ASSERT_RELEASE(cset2.size() == 3); + YGM_ASSERT_RELEASE(cset2.count("dog") == (size_t)world.size() * 3); + YGM_ASSERT_RELEASE(cset2.count("cat") == (size_t)world.size() * 2); + YGM_ASSERT_RELEASE(cset2.count("bird") == (size_t)world.size()); + YGM_ASSERT_RELEASE(cset2.count("red") == 0); + YGM_ASSERT_RELEASE(cset2.size() == 3); } return 0; diff --git a/test/test_csv_headers.cpp b/test/test_csv_headers.cpp index c892574f..8f331fbc 100644 --- a/test/test_csv_headers.cpp +++ b/test/test_csv_headers.cpp @@ -17,16 +17,16 @@ int main(int argc, char** argv) { csvp.read_headers(); csvp.for_all([&world](const auto& vfields) { // Test lookups by header names - ASSERT_RELEASE(vfields["zero"].as_integer() == 0); - ASSERT_RELEASE(vfields["two"].as_integer() == 2); - ASSERT_RELEASE(vfields["four"].as_integer() == 4); - ASSERT_RELEASE(vfields["six"].as_integer() == 6); + YGM_ASSERT_RELEASE(vfields["zero"].as_integer() == 0); + YGM_ASSERT_RELEASE(vfields["two"].as_integer() == 2); + YGM_ASSERT_RELEASE(vfields["four"].as_integer() == 4); + YGM_ASSERT_RELEASE(vfields["six"].as_integer() == 6); // Test lookup by column names agrees with positional lookups - ASSERT_RELEASE(vfields["zero"].as_integer() == vfields[0].as_integer()); - ASSERT_RELEASE(vfields["two"].as_integer() == vfields[2].as_integer()); - ASSERT_RELEASE(vfields["four"].as_integer() == vfields[1].as_integer()); - ASSERT_RELEASE(vfields["six"].as_integer() == vfields[3].as_integer()); + YGM_ASSERT_RELEASE(vfields["zero"].as_integer() == vfields[0].as_integer()); + YGM_ASSERT_RELEASE(vfields["two"].as_integer() == vfields[2].as_integer()); + YGM_ASSERT_RELEASE(vfields["four"].as_integer() == vfields[1].as_integer()); + YGM_ASSERT_RELEASE(vfields["six"].as_integer() == vfields[3].as_integer()); }); world.barrier(); diff --git a/test/test_csv_parser.cpp b/test/test_csv_parser.cpp index f654a292..23ec7fc3 100644 --- a/test/test_csv_parser.cpp +++ b/test/test_csv_parser.cpp @@ -16,13 +16,13 @@ int main(int argc, char** argv) { ygm::io::csv_parser csvp(world, std::vector{"data/100.csv"}); csvp.for_all([&world, &local_count](const auto& vfields) { for (auto f : vfields) { - ASSERT_RELEASE(f.is_integer()); + YGM_ASSERT_RELEASE(f.is_integer()); local_count += f.as_integer(); } }); world.barrier(); - ASSERT_RELEASE(world.all_reduce_sum(local_count) == 100); + YGM_ASSERT_RELEASE(world.all_reduce_sum(local_count) == 100); return 0; } diff --git a/test/test_daily_output.cpp b/test/test_daily_output.cpp index 5584c6a7..b32a06eb 100644 --- a/test/test_daily_output.cpp +++ b/test/test_daily_output.cpp @@ -31,7 +31,7 @@ int main(int argc, char **argv) { if (world.rank0()) { std::string expected_path(prefix_path + "1970/1/1"); - ASSERT_RELEASE(fs::exists(fs::path(expected_path))); + YGM_ASSERT_RELEASE(fs::exists(fs::path(expected_path))); fs::remove_all(fs::path(base_dir)); } } @@ -77,7 +77,7 @@ int main(int argc, char **argv) { fs::remove_all(fs::path(base_dir)); } - ASSERT_RELEASE(xor_write == xor_read); + YGM_ASSERT_RELEASE(xor_write == xor_read); } // Test appending @@ -132,7 +132,7 @@ int main(int argc, char **argv) { fs::remove_all(fs::path(base_dir)); } - ASSERT_RELEASE(xor_write == xor_read); + YGM_ASSERT_RELEASE(xor_write == xor_read); } return 0; diff --git a/test/test_disjoint_set.cpp b/test/test_disjoint_set.cpp index 1e54209c..20c4794d 100644 --- a/test/test_disjoint_set.cpp +++ b/test/test_disjoint_set.cpp @@ -45,8 +45,8 @@ int main(int argc, char** argv) { std::vector to_find = {"cat", "dog", "car"}; auto reps = dset.all_find(to_find); - ASSERT_RELEASE(reps["cat"] == reps["dog"]); - ASSERT_RELEASE(reps["cat"] != reps["car"]); + YGM_ASSERT_RELEASE(reps["cat"] == reps["dog"]); + YGM_ASSERT_RELEASE(reps["cat"] != reps["car"]); } // @@ -67,8 +67,8 @@ int main(int argc, char** argv) { std::vector to_find = {"cat", "dog", "car"}; auto reps = dset.all_find(to_find); - ASSERT_RELEASE(reps["cat"] == reps["dog"]); - ASSERT_RELEASE(reps["cat"] != reps["car"]); + YGM_ASSERT_RELEASE(reps["cat"] == reps["dog"]); + YGM_ASSERT_RELEASE(reps["cat"] != reps["car"]); } // @@ -86,13 +86,13 @@ int main(int argc, char** argv) { dset.async_union("cat", "dog"); - ASSERT_RELEASE(dset.size() == 3); - ASSERT_RELEASE(dset.num_sets() == 2); + YGM_ASSERT_RELEASE(dset.size() == 3); + YGM_ASSERT_RELEASE(dset.num_sets() == 2); dset.clear(); - ASSERT_RELEASE(dset.size() == 0); - ASSERT_RELEASE(dset.num_sets() == 0); + YGM_ASSERT_RELEASE(dset.size() == 0); + YGM_ASSERT_RELEASE(dset.num_sets() == 0); } // @@ -110,7 +110,7 @@ int main(int argc, char** argv) { } world.barrier(); - ASSERT_RELEASE(dset.num_sets() == 6); + YGM_ASSERT_RELEASE(dset.num_sets() == 6); std::vector to_find = {0, 1, 2, 3, 4, 5}; @@ -120,24 +120,24 @@ int main(int argc, char** argv) { dset.async_union(3, 4); dset.async_union(4, 5); - ASSERT_RELEASE(dset.num_sets() == 2); + YGM_ASSERT_RELEASE(dset.num_sets() == 2); auto reps = dset.all_find(to_find); - ASSERT_RELEASE(reps[0] == reps[1]); - ASSERT_RELEASE(reps[1] == reps[2]); - ASSERT_RELEASE(reps[2] != reps[3]); - ASSERT_RELEASE(reps[3] == reps[4]); - ASSERT_RELEASE(reps[4] == reps[5]); + YGM_ASSERT_RELEASE(reps[0] == reps[1]); + YGM_ASSERT_RELEASE(reps[1] == reps[2]); + YGM_ASSERT_RELEASE(reps[2] != reps[3]); + YGM_ASSERT_RELEASE(reps[3] == reps[4]); + YGM_ASSERT_RELEASE(reps[4] == reps[5]); dset.async_union(0, 3); - ASSERT_RELEASE(dset.num_sets() == 1); + YGM_ASSERT_RELEASE(dset.num_sets() == 1); auto reps_final = dset.all_find(to_find); - ASSERT_RELEASE(reps_final[0] == reps_final[1]); - ASSERT_RELEASE(reps_final[1] == reps_final[2]); - ASSERT_RELEASE(reps_final[2] == reps_final[3]); - ASSERT_RELEASE(reps_final[3] == reps_final[4]); - ASSERT_RELEASE(reps_final[4] == reps_final[5]); + YGM_ASSERT_RELEASE(reps_final[0] == reps_final[1]); + YGM_ASSERT_RELEASE(reps_final[1] == reps_final[2]); + YGM_ASSERT_RELEASE(reps_final[2] == reps_final[3]); + YGM_ASSERT_RELEASE(reps_final[3] == reps_final[4]); + YGM_ASSERT_RELEASE(reps_final[4] == reps_final[5]); } // @@ -155,7 +155,7 @@ int main(int argc, char** argv) { } world.barrier(); - ASSERT_RELEASE(dset.num_sets() == 6); + YGM_ASSERT_RELEASE(dset.num_sets() == 6); std::vector to_find = {0, 1, 2, 3, 4, 5}; @@ -165,26 +165,26 @@ int main(int argc, char** argv) { dset.async_union(4, 5); dset.async_union(3, 5); - ASSERT_RELEASE(dset.num_sets() == 2); + YGM_ASSERT_RELEASE(dset.num_sets() == 2); auto reps = dset.all_find(to_find); - ASSERT_RELEASE(reps[0] == reps[1]); - ASSERT_RELEASE(reps[1] == reps[2]); - ASSERT_RELEASE(reps[2] != reps[3]); - ASSERT_RELEASE(reps[3] == reps[4]); - ASSERT_RELEASE(reps[4] == reps[5]); + YGM_ASSERT_RELEASE(reps[0] == reps[1]); + YGM_ASSERT_RELEASE(reps[1] == reps[2]); + YGM_ASSERT_RELEASE(reps[2] != reps[3]); + YGM_ASSERT_RELEASE(reps[3] == reps[4]); + YGM_ASSERT_RELEASE(reps[4] == reps[5]); dset.async_union(0, 3); - ASSERT_RELEASE(dset.num_sets() == 1); + YGM_ASSERT_RELEASE(dset.num_sets() == 1); dset.all_compress(); auto reps_final = dset.all_find(to_find); - ASSERT_RELEASE(reps_final[0] == reps_final[1]); - ASSERT_RELEASE(reps_final[1] == reps_final[2]); - ASSERT_RELEASE(reps_final[2] == reps_final[3]); - ASSERT_RELEASE(reps_final[3] == reps_final[4]); - ASSERT_RELEASE(reps_final[4] == reps_final[5]); + YGM_ASSERT_RELEASE(reps_final[0] == reps_final[1]); + YGM_ASSERT_RELEASE(reps_final[1] == reps_final[2]); + YGM_ASSERT_RELEASE(reps_final[2] == reps_final[3]); + YGM_ASSERT_RELEASE(reps_final[3] == reps_final[4]); + YGM_ASSERT_RELEASE(reps_final[4] == reps_final[5]); } // @@ -200,11 +200,11 @@ int main(int argc, char** argv) { } dset.for_all([&counter](const auto& item, const auto& rep) { - ASSERT_RELEASE(item == rep); + YGM_ASSERT_RELEASE(item == rep); ++counter; }); - ASSERT_RELEASE(world.all_reduce_sum(counter) == num_items); + YGM_ASSERT_RELEASE(world.all_reduce_sum(counter) == num_items); } // Test async_union_and_execute @@ -224,6 +224,6 @@ int main(int argc, char** argv) { world.barrier(); - ASSERT_RELEASE(world.all_reduce_sum(counter) == 3); + YGM_ASSERT_RELEASE(world.all_reduce_sum(counter) == 3); } } diff --git a/test/test_interrupt_mask.cpp b/test/test_interrupt_mask.cpp index 18fb15c5..5bb29eff 100644 --- a/test/test_interrupt_mask.cpp +++ b/test/test_interrupt_mask.cpp @@ -24,13 +24,13 @@ int main(int argc, char** argv) { world.cf_barrier(); - ASSERT_RELEASE(count == 0); + YGM_ASSERT_RELEASE(count == 0); } world.barrier(); if (world.rank0()) { - ASSERT_RELEASE(count == num_sends * world.size()); + YGM_ASSERT_RELEASE(count == num_sends * world.size()); } return 0; diff --git a/test/test_large_messages.cpp b/test/test_large_messages.cpp index 73c2ebcd..a8b55335 100644 --- a/test/test_large_messages.cpp +++ b/test/test_large_messages.cpp @@ -34,7 +34,7 @@ int main(int argc, char** argv) { } world.barrier(); - ASSERT_RELEASE(counter == large_msg_size); + YGM_ASSERT_RELEASE(counter == large_msg_size); } return 0; diff --git a/test/test_layout.cpp b/test/test_layout.cpp index 6e5e8f3c..df6392e7 100644 --- a/test/test_layout.cpp +++ b/test/test_layout.cpp @@ -16,7 +16,7 @@ int main(int argc, char** argv) { int node_size(world.layout().node_size()); int min_node_size = world.all_reduce_min(node_size); world.barrier(); - ASSERT_RELEASE(min_node_size == node_size); + YGM_ASSERT_RELEASE(min_node_size == node_size); } // @@ -25,7 +25,7 @@ int main(int argc, char** argv) { int local_size(world.layout().local_size()); int min_local_size = world.all_reduce_min(local_size); world.barrier(); - ASSERT_RELEASE(min_local_size == local_size); + YGM_ASSERT_RELEASE(min_local_size == local_size); } // @@ -33,8 +33,8 @@ int main(int argc, char** argv) { { for (int dst(0); dst < world.size(); ++dst) { auto p = world.layout().rank_to_nl(dst); - ASSERT_RELEASE(p.first == world.layout().node_id(dst)); - ASSERT_RELEASE(p.second == world.layout().local_id(dst)); + YGM_ASSERT_RELEASE(p.first == world.layout().node_id(dst)); + YGM_ASSERT_RELEASE(p.second == world.layout().local_id(dst)); } world.barrier(); } @@ -45,8 +45,8 @@ int main(int argc, char** argv) { if (world.rank0()) { for (int dst(0); dst < world.size(); ++dst) { auto check_fn = [](auto pcomm, int node_guess, int local_guess) { - ASSERT_RELEASE(pcomm->layout().node_id() == node_guess); - ASSERT_RELEASE(pcomm->layout().local_id() == local_guess); + YGM_ASSERT_RELEASE(pcomm->layout().node_id() == node_guess); + YGM_ASSERT_RELEASE(pcomm->layout().local_id() == local_guess); }; auto p = world.layout().rank_to_nl(dst); world.async(dst, check_fn, p.first, p.second); @@ -59,7 +59,7 @@ int main(int argc, char** argv) { // is_local() is correct { auto check_fn = [](auto pcomm, int rank, bool tru) { - ASSERT_RELEASE(pcomm->layout().is_local(rank) == tru); + YGM_ASSERT_RELEASE(pcomm->layout().is_local(rank) == tru); }; bool target = (world.layout().node_id() == 0) ? true : false; @@ -72,9 +72,9 @@ int main(int argc, char** argv) { { std::vector strided_ranks = world.layout().strided_ranks(); for (auto sr : strided_ranks) { - ASSERT_RELEASE(world.layout().is_strided(sr) == true); + YGM_ASSERT_RELEASE(world.layout().is_strided(sr) == true); if (world.layout().rank() != sr) { - ASSERT_RELEASE(world.layout().is_local(sr) == false); + YGM_ASSERT_RELEASE(world.layout().is_local(sr) == false); } } world.barrier(); @@ -85,9 +85,9 @@ int main(int argc, char** argv) { { std::vector local_ranks = world.layout().local_ranks(); for (auto lr : local_ranks) { - ASSERT_RELEASE(world.layout().is_local(lr) == true); + YGM_ASSERT_RELEASE(world.layout().is_local(lr) == true); if (world.layout().rank() != lr) { - ASSERT_RELEASE(world.layout().is_strided(lr) == false); + YGM_ASSERT_RELEASE(world.layout().is_strided(lr) == false); } } world.barrier(); @@ -96,9 +96,9 @@ int main(int argc, char** argv) { { std::vector strided_ranks = world.layout().strided_ranks(); auto check_fn = [](auto pcomm, int src_rank) { - ASSERT_RELEASE(pcomm->layout().is_strided(src_rank) == true); + YGM_ASSERT_RELEASE(pcomm->layout().is_strided(src_rank) == true); if (pcomm->layout().rank() != src_rank) { - ASSERT_RELEASE(pcomm->layout().is_local(src_rank) == false); + YGM_ASSERT_RELEASE(pcomm->layout().is_local(src_rank) == false); } }; for (auto dst : strided_ranks) { @@ -110,9 +110,9 @@ int main(int argc, char** argv) { { std::vector local_ranks = world.layout().local_ranks(); auto check_fn = [](auto pcomm, int src_rank) { - ASSERT_RELEASE(pcomm->layout().is_local(src_rank) == true); + YGM_ASSERT_RELEASE(pcomm->layout().is_local(src_rank) == true); if (pcomm->layout().rank() != src_rank) { - ASSERT_RELEASE(pcomm->layout().is_strided(src_rank) == false); + YGM_ASSERT_RELEASE(pcomm->layout().is_strided(src_rank) == false); } }; for (auto dst : local_ranks) { diff --git a/test/test_line_parser.cpp b/test/test_line_parser.cpp index e7700f43..2eb2174a 100644 --- a/test/test_line_parser.cpp +++ b/test/test_line_parser.cpp @@ -64,7 +64,7 @@ void test_line_parser_files(ygm::comm& comm, const std::vector& fil std::set line_set_sequential; for (const auto& f : files) { std::ifstream ifs(f.c_str()); - ASSERT_RELEASE(ifs.good()); + YGM_ASSERT_RELEASE(ifs.good()); std::string line; while (std::getline(ifs, line)) { line_set.async_insert(line); @@ -72,10 +72,10 @@ void test_line_parser_files(ygm::comm& comm, const std::vector& fil } } - ASSERT_RELEASE(line_set.size() == line_set_sequential.size()); + YGM_ASSERT_RELEASE(line_set.size() == line_set_sequential.size()); //comm.cout0(line_set.size(), " =? ", line_set_to_test.size()); - ASSERT_RELEASE(line_set.size() == line_set_to_test.size()); - // ASSERT_RELEASE(line_set == line_set_to_test); + YGM_ASSERT_RELEASE(line_set.size() == line_set_to_test.size()); + // YGM_ASSERT_RELEASE(line_set == line_set_to_test); } @@ -88,5 +88,5 @@ void test_line_parser_directory(ygm::comm& comm, const std::string& dir, size_t line_set_to_test.async_insert(line); }); - ASSERT_RELEASE(unique_line_count == line_set_to_test.size()); + YGM_ASSERT_RELEASE(unique_line_count == line_set_to_test.size()); } \ No newline at end of file diff --git a/test/test_map.cpp b/test/test_map.cpp index f7916635..821e3a3a 100644 --- a/test/test_map.cpp +++ b/test/test_map.cpp @@ -35,9 +35,9 @@ int main(int argc, char **argv) { smap.async_insert("apple", "orange"); smap.async_insert("red", "green"); } - ASSERT_RELEASE(smap.count("dog") == 1); - ASSERT_RELEASE(smap.count("apple") == 1); - ASSERT_RELEASE(smap.count("red") == 1); + YGM_ASSERT_RELEASE(smap.count("dog") == 1); + YGM_ASSERT_RELEASE(smap.count("apple") == 1); + YGM_ASSERT_RELEASE(smap.count("red") == 1); } // @@ -49,9 +49,9 @@ int main(int argc, char **argv) { smap.async_insert("apple", "orange"); smap.async_insert("red", "green"); - ASSERT_RELEASE(smap.count("dog") == 1); - ASSERT_RELEASE(smap.count("apple") == 1); - ASSERT_RELEASE(smap.count("red") == 1); + YGM_ASSERT_RELEASE(smap.count("dog") == 1); + YGM_ASSERT_RELEASE(smap.count("apple") == 1); + YGM_ASSERT_RELEASE(smap.count("red") == 1); } // @@ -70,20 +70,20 @@ int main(int argc, char **argv) { world.barrier(); smap.async_visit("dog", [](const auto &key, auto &value) { - ASSERT_RELEASE(value == "cat"); + YGM_ASSERT_RELEASE(value == "cat"); }); smap.async_visit_if_contains("apple", [](auto key, auto &value) { - ASSERT_RELEASE(value == "orange"); + YGM_ASSERT_RELEASE(value == "orange"); }); const ygm::container::map &csmap = smap; csmap.async_visit_if_contains( - "red", [](auto key, auto &value) { ASSERT_RELEASE(value == "green"); }); + "red", [](auto key, auto &value) { YGM_ASSERT_RELEASE(value == "green"); }); smap.async_visit_if_contains( "SHOULD_BE_MISSING", - [](auto key, auto &value) { ASSERT_RELEASE(false); }); + [](auto key, auto &value) { YGM_ASSERT_RELEASE(false); }); } // @@ -92,31 +92,31 @@ int main(int argc, char **argv) { ygm::container::map smap(world); smap.async_visit("dog", [](const std::string &key, const std::string &value) { - ASSERT_RELEASE(key == "dog"); - ASSERT_RELEASE(value == ""); + YGM_ASSERT_RELEASE(key == "dog"); + YGM_ASSERT_RELEASE(value == ""); }); smap.async_visit("cat", [](const std::string &key, std::string &value) { - ASSERT_RELEASE(key == "cat"); - ASSERT_RELEASE(value == ""); + YGM_ASSERT_RELEASE(key == "cat"); + YGM_ASSERT_RELEASE(value == ""); }); smap.async_visit_if_contains( - "red", [](const auto &k, const auto &v) { ASSERT_RELEASE(false); }); + "red", [](const auto &k, const auto &v) { YGM_ASSERT_RELEASE(false); }); - ASSERT_RELEASE(smap.count("dog") == 1); - ASSERT_RELEASE(smap.count("cat") == 1); - ASSERT_RELEASE(smap.count("red") == 0); + YGM_ASSERT_RELEASE(smap.count("dog") == 1); + YGM_ASSERT_RELEASE(smap.count("cat") == 1); + YGM_ASSERT_RELEASE(smap.count("red") == 0); - ASSERT_RELEASE(smap.size() == 2); + YGM_ASSERT_RELEASE(smap.size() == 2); if (world.rank() == 0) { smap.async_erase("dog"); } - ASSERT_RELEASE(smap.count("dog") == 0); - ASSERT_RELEASE(smap.size() == 1); + YGM_ASSERT_RELEASE(smap.count("dog") == 0); + YGM_ASSERT_RELEASE(smap.size() == 1); smap.async_erase("cat"); - ASSERT_RELEASE(smap.count("cat") == 0); + YGM_ASSERT_RELEASE(smap.count("cat") == 0); - ASSERT_RELEASE(smap.size() == 0); + YGM_ASSERT_RELEASE(smap.size() == 0); } // // @@ -138,7 +138,7 @@ int main(int argc, char **argv) { // world.barrier(); - // ASSERT_RELEASE(world.all_reduce_sum(dog_visit_counter) == world.size()); + // YGM_ASSERT_RELEASE(world.all_reduce_sum(dog_visit_counter) == world.size()); // static int apple_visit_counter{0}; @@ -150,14 +150,14 @@ int main(int argc, char **argv) { // world.barrier(); - // ASSERT_RELEASE(world.all_reduce_sum(apple_visit_counter) == + // YGM_ASSERT_RELEASE(world.all_reduce_sum(apple_visit_counter) == // world.size() - 1); // if (world.rank0()) { // smap.async_insert_else_visit( // "red", "green", // [](const auto &key, const auto &value, const auto &new_value) { - // ASSERT_RELEASE(true == false); + // YGM_ASSERT_RELEASE(true == false); // }); // } // } @@ -183,14 +183,14 @@ int main(int argc, char **argv) { smap.for_all([&world, &num_reductions](const auto &key, const auto &value) { if (key == "sum") { - ASSERT_RELEASE(value == world.size() * num_reductions * + YGM_ASSERT_RELEASE(value == world.size() * num_reductions * (num_reductions - 1) / 2); } else if (key == "min") { - ASSERT_RELEASE(value == 0); + YGM_ASSERT_RELEASE(value == 0); } else if (key == "max") { - ASSERT_RELEASE(value == num_reductions - 1); + YGM_ASSERT_RELEASE(value == num_reductions - 1); } else { - ASSERT_RELEASE(false); + YGM_ASSERT_RELEASE(false); } }); } @@ -205,15 +205,15 @@ int main(int argc, char **argv) { smap2.async_insert("apple", "orange"); smap2.async_insert("red", "green"); smap2.swap(smap); - ASSERT_RELEASE(smap2.size() == 0); + YGM_ASSERT_RELEASE(smap2.size() == 0); } - ASSERT_RELEASE(smap.size() == 3); - ASSERT_RELEASE(smap.count("dog") == 1); - ASSERT_RELEASE(smap.count("apple") == 1); - ASSERT_RELEASE(smap.count("red") == 1); + YGM_ASSERT_RELEASE(smap.size() == 3); + YGM_ASSERT_RELEASE(smap.count("dog") == 1); + YGM_ASSERT_RELEASE(smap.count("apple") == 1); + YGM_ASSERT_RELEASE(smap.count("red") == 1); smap.async_insert_or_assign("car", "truck"); - ASSERT_RELEASE(smap.size() == 4); - ASSERT_RELEASE(smap.count("car") == 1); + YGM_ASSERT_RELEASE(smap.size() == 4); + YGM_ASSERT_RELEASE(smap.count("car") == 1); } // @@ -239,10 +239,10 @@ int main(int argc, char **argv) { auto gmap = smap.key_gather(gather_list); if (world.rank0()) { - ASSERT_RELEASE(gmap["foo"][0] == "bar"); - ASSERT_RELEASE(gmap["foo"][1] == "baz"); + YGM_ASSERT_RELEASE(gmap["foo"][0] == "bar"); + YGM_ASSERT_RELEASE(gmap["foo"][1] == "baz"); } else { - ASSERT_RELEASE(gmap["foo"].empty()); + YGM_ASSERT_RELEASE(gmap["foo"].empty()); } } @@ -260,9 +260,9 @@ int main(int argc, char **argv) { smap2.async_insert(key, value); }); - ASSERT_RELEASE(smap2.count("dog") == 1); - ASSERT_RELEASE(smap2.count("apple") == 1); - ASSERT_RELEASE(smap2.count("red") == 1); + YGM_ASSERT_RELEASE(smap2.count("dog") == 1); + YGM_ASSERT_RELEASE(smap2.count("apple") == 1); + YGM_ASSERT_RELEASE(smap2.count("red") == 1); } return 0; diff --git a/test/test_multi_output.cpp b/test/test_multi_output.cpp index 4490d1a9..74f03471 100644 --- a/test/test_multi_output.cpp +++ b/test/test_multi_output.cpp @@ -30,7 +30,7 @@ int main(int argc, char **argv) { } std::string expected_path(prefix_path + subpath); - ASSERT_RELEASE(fs::exists(fs::path(expected_path))); + YGM_ASSERT_RELEASE(fs::exists(fs::path(expected_path))); world.barrier(); @@ -79,7 +79,7 @@ int main(int argc, char **argv) { fs::remove_all(fs::path(base_dir)); } - ASSERT_RELEASE(xor_write == xor_read); + YGM_ASSERT_RELEASE(xor_write == xor_read); } // Test appending @@ -134,7 +134,7 @@ int main(int argc, char **argv) { fs::remove_all(fs::path(base_dir)); } - ASSERT_RELEASE(xor_write == xor_read); + YGM_ASSERT_RELEASE(xor_write == xor_read); } return 0; diff --git a/test/test_multimap.cpp b/test/test_multimap.cpp index 054a8303..74140af4 100644 --- a/test/test_multimap.cpp +++ b/test/test_multimap.cpp @@ -20,9 +20,9 @@ int main(int argc, char **argv) { smap.async_insert("apple", "orange"); smap.async_insert("red", "green"); } - ASSERT_RELEASE(smap.count("dog") == 1); - ASSERT_RELEASE(smap.count("apple") == 1); - ASSERT_RELEASE(smap.count("red") == 1); + YGM_ASSERT_RELEASE(smap.count("dog") == 1); + YGM_ASSERT_RELEASE(smap.count("apple") == 1); + YGM_ASSERT_RELEASE(smap.count("red") == 1); } // @@ -34,9 +34,9 @@ int main(int argc, char **argv) { smap.async_insert("apple", "orange"); smap.async_insert("red", "green"); - ASSERT_RELEASE(smap.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(smap.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(smap.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.count("red") == (size_t)world.size()); } // @@ -45,33 +45,33 @@ int main(int argc, char **argv) { ygm::container::multimap smap(world); smap.async_visit("dog", [](const std::string &key, const std::string &value) { - ASSERT_RELEASE(key == "dog"); - ASSERT_RELEASE(value == ""); + YGM_ASSERT_RELEASE(key == "dog"); + YGM_ASSERT_RELEASE(value == ""); }); smap.async_visit("cat", [](const std::string &key, const std::string &value) { - ASSERT_RELEASE(key == "cat"); - ASSERT_RELEASE(value == ""); + YGM_ASSERT_RELEASE(key == "cat"); + YGM_ASSERT_RELEASE(value == ""); }); smap.async_visit_if_contains("red", [](const auto &key, const auto &value) { - ASSERT_RELEASE(false); + YGM_ASSERT_RELEASE(false); }); - ASSERT_RELEASE(smap.count("dog") == 1); - ASSERT_RELEASE(smap.count("cat") == 1); - ASSERT_RELEASE(smap.count("red") == 0); + YGM_ASSERT_RELEASE(smap.count("dog") == 1); + YGM_ASSERT_RELEASE(smap.count("cat") == 1); + YGM_ASSERT_RELEASE(smap.count("red") == 0); - ASSERT_RELEASE(smap.size() == 2); + YGM_ASSERT_RELEASE(smap.size() == 2); if (world.rank() == 0) { smap.async_erase("dog"); } - ASSERT_RELEASE(smap.count("dog") == 0); - ASSERT_RELEASE(smap.size() == 1); + YGM_ASSERT_RELEASE(smap.count("dog") == 0); + YGM_ASSERT_RELEASE(smap.size() == 1); smap.async_erase("cat"); - ASSERT_RELEASE(smap.count("cat") == 0); + YGM_ASSERT_RELEASE(smap.count("cat") == 0); - ASSERT_RELEASE(smap.size() == 0); + YGM_ASSERT_RELEASE(smap.size() == 0); } // @@ -79,31 +79,31 @@ int main(int argc, char **argv) { { ygm::container::multimap smap(world); smap.async_visit("dog", [](const std::string &key, std::string &value) { - ASSERT_RELEASE(key == "dog"); - ASSERT_RELEASE(value == ""); + YGM_ASSERT_RELEASE(key == "dog"); + YGM_ASSERT_RELEASE(value == ""); }); smap.async_visit("cat", [](const std::string &key, std::string &value) { - ASSERT_RELEASE(key == "cat"); - ASSERT_RELEASE(value == ""); + YGM_ASSERT_RELEASE(key == "cat"); + YGM_ASSERT_RELEASE(value == ""); }); smap.async_visit_if_contains( - "red", [](const auto &k, const auto &v) { ASSERT_RELEASE(false); }); + "red", [](const auto &k, const auto &v) { YGM_ASSERT_RELEASE(false); }); - ASSERT_RELEASE(smap.count("dog") == 1); - ASSERT_RELEASE(smap.count("cat") == 1); - ASSERT_RELEASE(smap.count("red") == 0); + YGM_ASSERT_RELEASE(smap.count("dog") == 1); + YGM_ASSERT_RELEASE(smap.count("cat") == 1); + YGM_ASSERT_RELEASE(smap.count("red") == 0); - ASSERT_RELEASE(smap.size() == 2); + YGM_ASSERT_RELEASE(smap.size() == 2); if (world.rank() == 0) { smap.async_erase("dog"); } - ASSERT_RELEASE(smap.count("dog") == 0); - ASSERT_RELEASE(smap.size() == 1); + YGM_ASSERT_RELEASE(smap.count("dog") == 0); + YGM_ASSERT_RELEASE(smap.size() == 1); smap.async_erase("cat"); - ASSERT_RELEASE(smap.count("cat") == 0); + YGM_ASSERT_RELEASE(smap.count("cat") == 0); - ASSERT_RELEASE(smap.size() == 0); + YGM_ASSERT_RELEASE(smap.size() == 0); } // // @@ -120,12 +120,12 @@ int main(int argc, char **argv) { // smap.async_visit_group( // "dog", [](auto pmap, const auto begin, const auto end) { - // ASSERT_RELEASE(std::distance(begin, end) == 2 * + // YGM_ASSERT_RELEASE(std::distance(begin, end) == 2 * // pmap->comm().size()); // }); // smap.async_visit_group( // "cat", [](auto pmap, const auto begin, const auto end) { - // ASSERT_RELEASE(std::distance(begin, end) == pmap->comm().size()); + // YGM_ASSERT_RELEASE(std::distance(begin, end) == pmap->comm().size()); // }); // } @@ -139,15 +139,15 @@ int main(int argc, char **argv) { smap2.async_insert("apple", "orange"); smap2.async_insert("red", "green"); smap2.swap(smap); - ASSERT_RELEASE(smap2.size() == 0); + YGM_ASSERT_RELEASE(smap2.size() == 0); } - ASSERT_RELEASE(smap.size() == 3 * (size_t)world.size()); - ASSERT_RELEASE(smap.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(smap.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(smap.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.size() == 3 * (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.count("red") == (size_t)world.size()); smap.async_insert("car", "truck"); - ASSERT_RELEASE(smap.size() == 4 * (size_t)world.size()); - ASSERT_RELEASE(smap.count("car") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.size() == 4 * (size_t)world.size()); + YGM_ASSERT_RELEASE(smap.count("car") == (size_t)world.size()); } // @@ -161,9 +161,9 @@ int main(int argc, char **argv) { world.barrier(); auto values = smap.local_get("foo"); if (smap.partitioner.owner("foo") == world.rank()) { - ASSERT_RELEASE(values.size() == 4 * (size_t)world.size()); + YGM_ASSERT_RELEASE(values.size() == 4 * (size_t)world.size()); } else { - ASSERT_RELEASE(values.size() == 0); + YGM_ASSERT_RELEASE(values.size() == 0); } } @@ -181,9 +181,9 @@ int main(int argc, char **argv) { smap2.async_insert(key, value); }); - ASSERT_RELEASE(smap2.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(smap2.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(smap2.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap2.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap2.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap2.count("red") == (size_t)world.size()); } // @@ -200,9 +200,9 @@ int main(int argc, char **argv) { smap2.async_insert(std::make_pair(k, v)); }); - ASSERT_RELEASE(smap2.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(smap2.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(smap2.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap2.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap2.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(smap2.count("red") == (size_t)world.size()); } return 0; diff --git a/test/test_multiset.cpp b/test/test_multiset.cpp index 31f2c86d..f04b315c 100644 --- a/test/test_multiset.cpp +++ b/test/test_multiset.cpp @@ -22,20 +22,20 @@ int main(int argc, char** argv) { sset.async_insert("apple"); sset.async_insert("red"); } - ASSERT_RELEASE(sset.count("dog") == 2); - ASSERT_RELEASE(sset.count("apple") == 1); - ASSERT_RELEASE(sset.count("red") == 1); - ASSERT_RELEASE(sset.size() == 4); + YGM_ASSERT_RELEASE(sset.count("dog") == 2); + YGM_ASSERT_RELEASE(sset.count("apple") == 1); + YGM_ASSERT_RELEASE(sset.count("red") == 1); + YGM_ASSERT_RELEASE(sset.size() == 4); if (world.rank() == 0) { sset.async_erase("dog"); } - ASSERT_RELEASE(sset.size() == 2); + YGM_ASSERT_RELEASE(sset.size() == 2); if (world.rank() == 0) { sset.async_erase("apple"); } - ASSERT_RELEASE(sset.size() == 1); - ASSERT_RELEASE(sset.count("dog") == 0); - ASSERT_RELEASE(sset.count("apple") == 0); + YGM_ASSERT_RELEASE(sset.size() == 1); + YGM_ASSERT_RELEASE(sset.count("dog") == 0); + YGM_ASSERT_RELEASE(sset.count("apple") == 0); } // @@ -47,12 +47,12 @@ int main(int argc, char** argv) { sset.async_insert("apple"); sset.async_insert("red"); - ASSERT_RELEASE(sset.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(sset.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(sset.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.count("red") == (size_t)world.size()); sset.async_insert("dog"); - ASSERT_RELEASE(sset.count("dog") == (size_t)world.size() * 2); + YGM_ASSERT_RELEASE(sset.count("dog") == (size_t)world.size() * 2); } // @@ -71,7 +71,7 @@ int main(int argc, char** argv) { iset.async_contains(val, f); } world.barrier(); - ASSERT_RELEASE(not ygm::logical_or(set_contains, world)); + YGM_ASSERT_RELEASE(not ygm::logical_or(set_contains, world)); if (world.rank0()) { iset.async_insert(val); @@ -81,7 +81,7 @@ int main(int argc, char** argv) { iset.async_contains(val, f); } world.barrier(); - ASSERT_RELEASE(ygm::logical_or(set_contains, world)); + YGM_ASSERT_RELEASE(ygm::logical_or(set_contains, world)); } // @@ -99,13 +99,13 @@ int main(int argc, char** argv) { sset.async_insert_contains("dog", f); } world.barrier(); - ASSERT_RELEASE(not ygm::logical_or(already_contains, world)); + YGM_ASSERT_RELEASE(not ygm::logical_or(already_contains, world)); if (world.rank0()) { sset.async_insert_contains("dog", f); } world.barrier(); - ASSERT_RELEASE(ygm::logical_or(already_contains, world)); + YGM_ASSERT_RELEASE(ygm::logical_or(already_contains, world)); } // @@ -118,15 +118,15 @@ int main(int argc, char** argv) { sset2.async_insert("apple"); sset2.async_insert("red"); sset2.swap(sset); - ASSERT_RELEASE(sset2.size() == 0); + YGM_ASSERT_RELEASE(sset2.size() == 0); } - ASSERT_RELEASE(sset.size() == 3 * (size_t)world.size()); - ASSERT_RELEASE(sset.count("dog") == (size_t)world.size()); - ASSERT_RELEASE(sset.count("apple") == (size_t)world.size()); - ASSERT_RELEASE(sset.count("red") == (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.size() == 3 * (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.count("dog") == (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.count("apple") == (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.count("red") == (size_t)world.size()); sset.async_insert("car"); - ASSERT_RELEASE(sset.size() == 4 * (size_t)world.size()); - ASSERT_RELEASE(sset.count("car") == (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.size() == 4 * (size_t)world.size()); + YGM_ASSERT_RELEASE(sset.count("car") == (size_t)world.size()); } // @@ -141,9 +141,9 @@ int main(int argc, char** argv) { sset1.for_all([&sset2](const auto &key) { sset2.async_insert(key); }); - ASSERT_RELEASE(sset2.count("dog") == world.size()); - ASSERT_RELEASE(sset2.count("apple") == world.size()); - ASSERT_RELEASE(sset2.count("red") == world.size()); + YGM_ASSERT_RELEASE(sset2.count("dog") == world.size()); + YGM_ASSERT_RELEASE(sset2.count("apple") == world.size()); + YGM_ASSERT_RELEASE(sset2.count("red") == world.size()); } // @@ -164,7 +164,7 @@ int main(int argc, char** argv) { world.barrier(); for (int set_index = 0; set_index < num_sets; ++set_index) { - ASSERT_RELEASE(vec_sets[set_index].size() == world.size() * 2); + YGM_ASSERT_RELEASE(vec_sets[set_index].size() == world.size() * 2); } } diff --git a/test/test_ndjson_parser.cpp b/test/test_ndjson_parser.cpp index 1f1c9923..d017fedd 100644 --- a/test/test_ndjson_parser.cpp +++ b/test/test_ndjson_parser.cpp @@ -18,7 +18,7 @@ int main(int argc, char** argv) { jsonp.for_all([&world, &local_count](const auto& json) { ++local_count; }); world.barrier(); - ASSERT_RELEASE(world.all_reduce_sum(local_count) == 3); + YGM_ASSERT_RELEASE(world.all_reduce_sum(local_count) == 3); return 0; } diff --git a/test/test_parquet_reader.cpp b/test/test_parquet_reader.cpp index aac2aa2e..f55c3548 100644 --- a/test/test_parquet_reader.cpp +++ b/test/test_parquet_reader.cpp @@ -33,7 +33,7 @@ int main(int argc, char** argv) { world.barrier(); auto row_count = world.all_reduce_sum(local_count); - ASSERT_RELEASE(row_count == 12); + YGM_ASSERT_RELEASE(row_count == 12); } // @@ -74,9 +74,9 @@ int main(int argc, char** argv) { world.barrier(); auto row_count = world.all_reduce_sum(rows.size()); - ASSERT_RELEASE(row_count == 12); + YGM_ASSERT_RELEASE(row_count == 12); - ASSERT_RELEASE(world.all_reduce_sum(strings.count("Hennessey Venom F5")) == + YGM_ASSERT_RELEASE(world.all_reduce_sum(strings.count("Hennessey Venom F5")) == 1); } @@ -126,9 +126,9 @@ int main(int argc, char** argv) { world.barrier(); const auto sum = world.all_reduce_sum(local_sum); - ASSERT_RELEASE(sum == 11111111111); + YGM_ASSERT_RELEASE(sum == 11111111111); const auto row_count = world.all_reduce_sum(local_count); - ASSERT_RELEASE(row_count == 11); + YGM_ASSERT_RELEASE(row_count == 11); } return 0; diff --git a/test/test_parquet_reader_json.cpp b/test/test_parquet_reader_json.cpp index 0ce0b5d3..0e8e4b14 100644 --- a/test/test_parquet_reader_json.cpp +++ b/test/test_parquet_reader_json.cpp @@ -32,49 +32,49 @@ int main(int argc, char** argv) { world.async( 0, [](auto, const auto& obj) { - ASSERT_RELEASE(obj.contains("id")); - ASSERT_RELEASE(obj.contains("bool")); - ASSERT_RELEASE(obj.contains("int32")); - ASSERT_RELEASE(obj.contains("int64")); - ASSERT_RELEASE(obj.contains("float")); - ASSERT_RELEASE(obj.contains("double")); - ASSERT_RELEASE(obj.contains("byte_array")); + YGM_ASSERT_RELEASE(obj.contains("id")); + YGM_ASSERT_RELEASE(obj.contains("bool")); + YGM_ASSERT_RELEASE(obj.contains("int32")); + YGM_ASSERT_RELEASE(obj.contains("int64")); + YGM_ASSERT_RELEASE(obj.contains("float")); + YGM_ASSERT_RELEASE(obj.contains("double")); + YGM_ASSERT_RELEASE(obj.contains("byte_array")); - ASSERT_RELEASE(obj.at("id").is_int64()); - ASSERT_RELEASE(obj.at("bool").is_bool()); - ASSERT_RELEASE(obj.at("int32").is_int64()); - ASSERT_RELEASE(obj.at("int64").is_int64()); - ASSERT_RELEASE(obj.at("float").is_double()); - ASSERT_RELEASE(obj.at("double").is_double()); - ASSERT_RELEASE(obj.at("byte_array").is_string()); + YGM_ASSERT_RELEASE(obj.at("id").is_int64()); + YGM_ASSERT_RELEASE(obj.at("bool").is_bool()); + YGM_ASSERT_RELEASE(obj.at("int32").is_int64()); + YGM_ASSERT_RELEASE(obj.at("int64").is_int64()); + YGM_ASSERT_RELEASE(obj.at("float").is_double()); + YGM_ASSERT_RELEASE(obj.at("double").is_double()); + YGM_ASSERT_RELEASE(obj.at("byte_array").is_string()); const auto id = obj.at("id").as_int64(); if (id == 0) { - ASSERT_RELEASE(obj.at("bool").as_bool() == true); - ASSERT_RELEASE(obj.at("int32").as_int64() == -1); - ASSERT_RELEASE(obj.at("int64").as_int64() == -(1ULL << 32) - 1); - ASSERT_RELEASE(obj.at("float").as_double() == 1.5); - ASSERT_RELEASE(obj.at("double").as_double() == 10.5); - ASSERT_RELEASE(obj.at("byte_array").as_string() == "aa"); + YGM_ASSERT_RELEASE(obj.at("bool").as_bool() == true); + YGM_ASSERT_RELEASE(obj.at("int32").as_int64() == -1); + YGM_ASSERT_RELEASE(obj.at("int64").as_int64() == -(1ULL << 32) - 1); + YGM_ASSERT_RELEASE(obj.at("float").as_double() == 1.5); + YGM_ASSERT_RELEASE(obj.at("double").as_double() == 10.5); + YGM_ASSERT_RELEASE(obj.at("byte_array").as_string() == "aa"); ++cnt1; } else if (id == 1) { - ASSERT_RELEASE(obj.at("bool").as_bool() == false); - ASSERT_RELEASE(obj.at("int32").as_int64() == -2); - ASSERT_RELEASE(obj.at("int64").as_int64() == -(1ULL << 32) - 2); - ASSERT_RELEASE(obj.at("float").as_double() == 2.5); - ASSERT_RELEASE(obj.at("double").as_double() == 20.5); - ASSERT_RELEASE(obj.at("byte_array").as_string() == "bb"); + YGM_ASSERT_RELEASE(obj.at("bool").as_bool() == false); + YGM_ASSERT_RELEASE(obj.at("int32").as_int64() == -2); + YGM_ASSERT_RELEASE(obj.at("int64").as_int64() == -(1ULL << 32) - 2); + YGM_ASSERT_RELEASE(obj.at("float").as_double() == 2.5); + YGM_ASSERT_RELEASE(obj.at("double").as_double() == 20.5); + YGM_ASSERT_RELEASE(obj.at("byte_array").as_string() == "bb"); ++cnt2; } else if (id == 2) { - ASSERT_RELEASE(obj.at("bool").as_bool() == true); - ASSERT_RELEASE(obj.at("int32").as_int64() == -3); - ASSERT_RELEASE(obj.at("int64").as_int64() == -(1ULL << 32) - 3); - ASSERT_RELEASE(obj.at("float").as_double() == 3.5); - ASSERT_RELEASE(obj.at("double").as_double() == 30.5); - ASSERT_RELEASE(obj.at("byte_array").as_string() == "cc"); + YGM_ASSERT_RELEASE(obj.at("bool").as_bool() == true); + YGM_ASSERT_RELEASE(obj.at("int32").as_int64() == -3); + YGM_ASSERT_RELEASE(obj.at("int64").as_int64() == -(1ULL << 32) - 3); + YGM_ASSERT_RELEASE(obj.at("float").as_double() == 3.5); + YGM_ASSERT_RELEASE(obj.at("double").as_double() == 30.5); + YGM_ASSERT_RELEASE(obj.at("byte_array").as_string() == "cc"); ++cnt3; } else { - ASSERT_RELEASE(false); + YGM_ASSERT_RELEASE(false); } }, obj); @@ -82,13 +82,13 @@ int main(int argc, char** argv) { world.barrier(); if (world.rank0()) { - ASSERT_RELEASE(cnt1 == 1); - ASSERT_RELEASE(cnt2 == 1); - ASSERT_RELEASE(cnt3 == 1); + YGM_ASSERT_RELEASE(cnt1 == 1); + YGM_ASSERT_RELEASE(cnt2 == 1); + YGM_ASSERT_RELEASE(cnt3 == 1); } else { - ASSERT_RELEASE(cnt1 == 0); - ASSERT_RELEASE(cnt2 == 0); - ASSERT_RELEASE(cnt3 == 0); + YGM_ASSERT_RELEASE(cnt1 == 0); + YGM_ASSERT_RELEASE(cnt2 == 0); + YGM_ASSERT_RELEASE(cnt3 == 0); } return 0; diff --git a/test/test_random.cpp b/test/test_random.cpp index dc8703f4..9f28e714 100644 --- a/test/test_random.cpp +++ b/test/test_random.cpp @@ -32,17 +32,17 @@ int main(int argc, char** argv) { int local_counter(0); seed_set.for_all([&local_counter](int key, int val) { - ASSERT_RELEASE(val == 1); + YGM_ASSERT_RELEASE(val == 1); ++local_counter; }); // this can fail if two samples collide, but that is very unlikely. // is it worth the trouble of making the test more robust? - rn_set.for_all([](int key, int val) { ASSERT_RELEASE(val == 1); }); - sample_set.for_all([](int key, int val) { ASSERT_RELEASE(val == 1); }); + rn_set.for_all([](int key, int val) { YGM_ASSERT_RELEASE(val == 1); }); + sample_set.for_all([](int key, int val) { YGM_ASSERT_RELEASE(val == 1); }); int global_counter = world.all_reduce_sum(local_counter); - ASSERT_RELEASE(global_counter == world.size()); + YGM_ASSERT_RELEASE(global_counter == world.size()); } } \ No newline at end of file diff --git a/test/test_recursion_large_messages.cpp b/test/test_recursion_large_messages.cpp index 03546cd8..2696cf82 100644 --- a/test/test_recursion_large_messages.cpp +++ b/test/test_recursion_large_messages.cpp @@ -49,7 +49,7 @@ int main(int argc, char **argv) { } world.barrier(); - ASSERT_RELEASE(ygm::sum(counter, world) == ((size_t(1) << max_hops) - 1)); + YGM_ASSERT_RELEASE(ygm::sum(counter, world) == ((size_t(1) << max_hops) - 1)); } return 0; diff --git a/test/test_recursion_progress.cpp b/test/test_recursion_progress.cpp index 59721447..f42a7ad0 100644 --- a/test/test_recursion_progress.cpp +++ b/test/test_recursion_progress.cpp @@ -39,7 +39,7 @@ int main(int argc, char **argv) { } world.barrier(); - ASSERT_RELEASE(ygm::sum(counter, world) == (world.size() * trips + 1)); + YGM_ASSERT_RELEASE(ygm::sum(counter, world) == (world.size() * trips + 1)); } return 0; diff --git a/test/test_reduce_by_key.cpp b/test/test_reduce_by_key.cpp index 1cfa0f22..b274dd54 100644 --- a/test/test_reduce_by_key.cpp +++ b/test/test_reduce_by_key.cpp @@ -20,9 +20,9 @@ int main(int argc, char** argv) { auto test = ygm::container::reduce_by_key_map( mybag, [](int a, int b) { return a + b; }, world); - ASSERT_RELEASE(test.size() == 1); + YGM_ASSERT_RELEASE(test.size() == 1); test.async_visit( - 0, [](int key, int value, int size) { ASSERT_RELEASE(value == size); }, + 0, [](int key, int value, int size) { YGM_ASSERT_RELEASE(value == size); }, world.size()); } @@ -35,21 +35,21 @@ int main(int argc, char** argv) { auto test = ygm::container::reduce_by_key_map( vec_str_count, [](size_t a, size_t b) { return a + b; }, world); - ASSERT_RELEASE(test.size() == 2); + YGM_ASSERT_RELEASE(test.size() == 2); size_t found = 0; test.for_all([&found, &world](const std::string& s, size_t c) { if (s == "Howdy") { ++found; - ASSERT_RELEASE(c == world.size()); + YGM_ASSERT_RELEASE(c == world.size()); } else if (s == "Aggs") { ++found; - ASSERT_RELEASE(c == world.size() * 2); + YGM_ASSERT_RELEASE(c == world.size() * 2); } else { - ASSERT_RELEASE(false); + YGM_ASSERT_RELEASE(false); } }); - ASSERT_RELEASE(world.all_reduce_sum(found) == 2); + YGM_ASSERT_RELEASE(world.all_reduce_sum(found) == 2); } return 0; diff --git a/test/test_reducing_adapter.cpp b/test/test_reducing_adapter.cpp index 4b2e71f4..01f65d53 100644 --- a/test/test_reducing_adapter.cpp +++ b/test/test_reducing_adapter.cpp @@ -44,12 +44,12 @@ int main(int argc, char **argv) { test_map.for_all( [&num_reductions, &world](const auto &key, const auto &value) { if (key == "max") { - ASSERT_RELEASE(value == num_reductions - 1); + YGM_ASSERT_RELEASE(value == num_reductions - 1); } else if (key == "sum") { - ASSERT_RELEASE(value == world.size() * num_reductions * + YGM_ASSERT_RELEASE(value == world.size() * num_reductions * (num_reductions - 1) / 2); } else { - ASSERT_RELEASE(false); + YGM_ASSERT_RELEASE(false); } }); } @@ -70,9 +70,9 @@ int main(int argc, char **argv) { test_array.for_all([&num_reductions](const auto &index, const auto &value) { if (index == 0) { - ASSERT_RELEASE(value == num_reductions - 1); + YGM_ASSERT_RELEASE(value == num_reductions - 1); } else { - ASSERT_RELEASE(value == 0); + YGM_ASSERT_RELEASE(value == 0); } }); } diff --git a/test/test_set.cpp b/test/test_set.cpp index dd74446c..17d4f4e1 100644 --- a/test/test_set.cpp +++ b/test/test_set.cpp @@ -36,10 +36,10 @@ int main(int argc, char **argv) { sset.async_insert("apple"); sset.async_insert("red"); } - ASSERT_RELEASE(sset.count("dog") == 1); - ASSERT_RELEASE(sset.count("red") == 1); - ASSERT_RELEASE(sset.count("apple") == 1); - ASSERT_RELEASE(sset.size() == 3); + YGM_ASSERT_RELEASE(sset.count("dog") == 1); + YGM_ASSERT_RELEASE(sset.count("red") == 1); + YGM_ASSERT_RELEASE(sset.count("apple") == 1); + YGM_ASSERT_RELEASE(sset.size() == 3); ygm::container::set iset(world); if (world.rank() == 0) { @@ -47,10 +47,10 @@ int main(int argc, char **argv) { iset.async_insert(7); iset.async_insert(100); } - ASSERT_RELEASE(iset.count(42) == 1); - ASSERT_RELEASE(iset.count(7) == 1); - ASSERT_RELEASE(iset.count(100) == 1); - ASSERT_RELEASE(iset.size() == 3); + YGM_ASSERT_RELEASE(iset.count(42) == 1); + YGM_ASSERT_RELEASE(iset.count(7) == 1); + YGM_ASSERT_RELEASE(iset.count(100) == 1); + YGM_ASSERT_RELEASE(iset.size() == 3); } // @@ -63,10 +63,10 @@ int main(int argc, char **argv) { sset_ptr->async_insert("apple"); sset_ptr->async_insert("red"); } - ASSERT_RELEASE(sset.count("dog") == 1); - ASSERT_RELEASE(sset.count("apple") == 1); - ASSERT_RELEASE(sset.count("red") == 1); - ASSERT_RELEASE(sset.size() == 3); + YGM_ASSERT_RELEASE(sset.count("dog") == 1); + YGM_ASSERT_RELEASE(sset.count("apple") == 1); + YGM_ASSERT_RELEASE(sset.count("red") == 1); + YGM_ASSERT_RELEASE(sset.size() == 3); } @@ -79,13 +79,13 @@ int main(int argc, char **argv) { sset.async_insert("apple"); sset.async_insert("red"); - ASSERT_RELEASE(sset.count("dog") == 1); - ASSERT_RELEASE(sset.count("apple") == 1); - ASSERT_RELEASE(sset.count("red") == 1); - ASSERT_RELEASE(sset.size() == 3); + YGM_ASSERT_RELEASE(sset.count("dog") == 1); + YGM_ASSERT_RELEASE(sset.count("apple") == 1); + YGM_ASSERT_RELEASE(sset.count("red") == 1); + YGM_ASSERT_RELEASE(sset.size() == 3); sset.async_erase("dog"); - ASSERT_RELEASE(sset.count("dog") == 0); - ASSERT_RELEASE(sset.size() == 2); + YGM_ASSERT_RELEASE(sset.count("dog") == 0); + YGM_ASSERT_RELEASE(sset.size() == 2); } @@ -104,7 +104,7 @@ int main(int argc, char **argv) { iset.async_contains(val, f); } world.barrier(); - ASSERT_RELEASE(not ygm::logical_or(set_contains, world)); + YGM_ASSERT_RELEASE(not ygm::logical_or(set_contains, world)); if (world.rank0()) { iset.async_insert(val); @@ -115,7 +115,7 @@ int main(int argc, char **argv) { iset.async_contains(val, f); } world.barrier(); - ASSERT_RELEASE(ygm::logical_or(set_contains, world)); + YGM_ASSERT_RELEASE(ygm::logical_or(set_contains, world)); } // @@ -132,36 +132,36 @@ int main(int argc, char **argv) { sset.async_insert_contains("dog", f); } world.barrier(); - ASSERT_RELEASE(not ygm::logical_or(did_contain, world)); + YGM_ASSERT_RELEASE(not ygm::logical_or(did_contain, world)); if (world.rank0()) { sset.async_insert_contains("dog", f); } world.barrier(); - ASSERT_RELEASE(ygm::logical_or(did_contain, world)); + YGM_ASSERT_RELEASE(ygm::logical_or(did_contain, world)); } // Test from bag { ygm::container::bag sbag(world, {"one", "two", "three", "one", "two"}); - ASSERT_RELEASE(sbag.size() == 5); + YGM_ASSERT_RELEASE(sbag.size() == 5); ygm::container::set sset(world, sbag); - ASSERT_RELEASE(sset.size() == 3); + YGM_ASSERT_RELEASE(sset.size() == 3); } // Test initializer list { ygm::container::set sset(world, {"one", "two", "three", "one", "two"}); - ASSERT_RELEASE(sset.size() == 3); + YGM_ASSERT_RELEASE(sset.size() == 3); } // Test from STL vector { std::vector v({1,2,3,4,5,1,1,1,3}); ygm::container::set iset(world, v); - ASSERT_RELEASE(iset.size() == 5); + YGM_ASSERT_RELEASE(iset.size() == 5); } @@ -185,15 +185,15 @@ int main(int argc, char **argv) { sset2.async_insert("apple"); sset2.async_insert("red"); sset2.swap(sset); - ASSERT_RELEASE(sset2.size() == 0); + YGM_ASSERT_RELEASE(sset2.size() == 0); } - ASSERT_RELEASE(sset.size() == 3); - ASSERT_RELEASE(sset.count("dog") == 1); - ASSERT_RELEASE(sset.count("apple") == 1); - ASSERT_RELEASE(sset.count("red") == 1); + YGM_ASSERT_RELEASE(sset.size() == 3); + YGM_ASSERT_RELEASE(sset.count("dog") == 1); + YGM_ASSERT_RELEASE(sset.count("apple") == 1); + YGM_ASSERT_RELEASE(sset.count("red") == 1); sset.async_insert("car"); - ASSERT_RELEASE(sset.size() == 4); - ASSERT_RELEASE(sset.count("car") == 1); + YGM_ASSERT_RELEASE(sset.size() == 4); + YGM_ASSERT_RELEASE(sset.count("car") == 1); } // @@ -208,9 +208,9 @@ int main(int argc, char **argv) { sset1.for_all([&sset2](const auto &key) { sset2.async_insert(key); }); - ASSERT_RELEASE(sset2.count("dog") == 1); - ASSERT_RELEASE(sset2.count("apple") == 1); - ASSERT_RELEASE(sset2.count("red") == 1); + YGM_ASSERT_RELEASE(sset2.count("dog") == 1); + YGM_ASSERT_RELEASE(sset2.count("apple") == 1); + YGM_ASSERT_RELEASE(sset2.count("red") == 1); } // // @@ -225,10 +225,10 @@ int main(int argc, char **argv) { // sset1.consume_all([&sset2](const auto &key) { sset2.async_insert(key); }); - // ASSERT_RELEASE(sset1.empty()); - // ASSERT_RELEASE(sset2.count("dog") == 1); - // ASSERT_RELEASE(sset2.count("apple") == 1); - // ASSERT_RELEASE(sset2.count("red") == 1); + // YGM_ASSERT_RELEASE(sset1.empty()); + // YGM_ASSERT_RELEASE(sset2.count("dog") == 1); + // YGM_ASSERT_RELEASE(sset2.count("apple") == 1); + // YGM_ASSERT_RELEASE(sset2.count("red") == 1); // } // // @@ -244,10 +244,10 @@ int main(int argc, char **argv) { // ygm::consume_all_iterative_adapter cai(sset1); // cai.consume_all([&sset2](const auto &key) { sset2.async_insert(key); }); - // ASSERT_RELEASE(sset1.empty()); - // ASSERT_RELEASE(sset2.count("dog") == 1); - // ASSERT_RELEASE(sset2.count("apple") == 1); - // ASSERT_RELEASE(sset2.count("red") == 1); + // YGM_ASSERT_RELEASE(sset1.empty()); + // YGM_ASSERT_RELEASE(sset2.count("dog") == 1); + // YGM_ASSERT_RELEASE(sset2.count("apple") == 1); + // YGM_ASSERT_RELEASE(sset2.count("red") == 1); // } // @@ -268,7 +268,7 @@ int main(int argc, char **argv) { world.barrier(); for (int set_index = 0; set_index < num_sets; ++set_index) { - ASSERT_RELEASE(vec_sets[set_index].size() == world.size() + 1); + YGM_ASSERT_RELEASE(vec_sets[set_index].size() == world.size() + 1); } } diff --git a/test/test_tagged_bag.cpp b/test/test_tagged_bag.cpp index 3947f782..3c5b8138 100644 --- a/test/test_tagged_bag.cpp +++ b/test/test_tagged_bag.cpp @@ -30,14 +30,14 @@ int main(int argc, char** argv) { r0tags = std::vector{r0t1, r0t2, r0t3}; } - ASSERT_RELEASE(tagbag.size() == 3); + YGM_ASSERT_RELEASE(tagbag.size() == 3); // Test gather auto gather = tagbag.key_gather(r0tags); world.barrier(); if (world.rank0()) { - ASSERT_RELEASE(gather.size() == 3); + YGM_ASSERT_RELEASE(gather.size() == 3); } else { - ASSERT_RELEASE(gather.empty()); + YGM_ASSERT_RELEASE(gather.empty()); } tagbag.for_all([](auto& k, auto& v) { v += "_added"; }); @@ -46,7 +46,7 @@ int main(int argc, char** argv) { if (world.rank0()) { for (auto r0tag : r0tags) { auto ga = gatheradd.at(r0tag); - ASSERT_RELEASE(ga.substr(ga.size() - 6) == "_added"); + YGM_ASSERT_RELEASE(ga.substr(ga.size() - 6) == "_added"); } } } @@ -58,6 +58,6 @@ int main(int argc, char** argv) { sbag.async_insert("dog"); sbag.async_insert("apple"); sbag.async_insert("red"); - ASSERT_RELEASE(sbag.size() == 3 * (size_t)world.size()); + YGM_ASSERT_RELEASE(sbag.size() == 3 * (size_t)world.size()); } } \ No newline at end of file From 6da544a914ba8ae5a4961018fe50ec2fad9d727c Mon Sep 17 00:00:00 2001 From: Roger Pearce Date: Fri, 2 Aug 2024 14:04:25 -0500 Subject: [PATCH 08/40] Hotfix (#230) --- include/ygm/container/counting_set.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/ygm/container/counting_set.hpp b/include/ygm/container/counting_set.hpp index e5992b7b..f6974b31 100644 --- a/include/ygm/container/counting_set.hpp +++ b/include/ygm/container/counting_set.hpp @@ -44,7 +44,7 @@ class counting_set counting_set() = delete; counting_set(ygm::comm &comm, std::initializer_list l) - : m_map(comm ), + : m_map(comm), m_comm(comm), partitioner(m_map.partitioner), pthis(this) { @@ -61,7 +61,7 @@ class counting_set counting_set(ygm::comm &comm, const STLContainer &cont) requires detail::STLContainer && std::convertible_to - : m_comm(comm), pthis(this), partitioner(comm) { + : m_map(comm), m_comm(comm), pthis(this), partitioner(comm) { pthis.check(m_comm); for (const Key &i : cont) { @@ -75,7 +75,7 @@ class counting_set requires detail::HasForAll && detail::SingleItemTuple< typename YGMContainer::for_all_args> - : m_comm(comm), pthis(this), partitioner(comm) { + : m_map(comm), m_comm(comm), pthis(this), partitioner(comm) { pthis.check(m_comm); yc.for_all([this](const Key &value) { this->async_insert(value); }); From 75e311b13da72779cb72a01d02ba3542072dc320 Mon Sep 17 00:00:00 2001 From: Trevor Steil Date: Fri, 2 Aug 2024 12:10:57 -0700 Subject: [PATCH 09/40] Feature/container docs (#229) * Updates container documentation * Fixes links in rtd/getting_started page --- docs/CMakeLists.txt | 2 +- docs/rtd/getting_started.rst | 3 +- docs/rtd/ygm/container.rst | 95 +++++++++++++++++++++++++++--------- 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 55a1d082..8265c559 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -29,4 +29,4 @@ doxygen_add_docs(doxygen COMMENT "Generate documentation by Doxygen" ) -add_subdirectory(rtd) \ No newline at end of file +add_subdirectory(rtd) diff --git a/docs/rtd/getting_started.rst b/docs/rtd/getting_started.rst index 9b4dcbe6..2b88bf84 100644 --- a/docs/rtd/getting_started.rst +++ b/docs/rtd/getting_started.rst @@ -106,7 +106,8 @@ YGM is distributed under the MIT license. All new contributions must be made under the MIT license. -See [LICENSE-MIT](LICENSE-MIT), [NOTICE](NOTICE), and [COPYRIGHT](COPYRIGHT) for +See `LICENSE-MIT `_, `NOTICE +`_, and `COPYRIGHT `_ for details. SPDX-License-Identifier: MIT diff --git a/docs/rtd/ygm/container.rst b/docs/rtd/ygm/container.rst index 05b3ba85..758c9b74 100644 --- a/docs/rtd/ygm/container.rst +++ b/docs/rtd/ygm/container.rst @@ -11,34 +11,68 @@ operations that need to be performed on the data stored in a container while abstracting the locality and access details of said data. While insiration is taken from STL, the top priority is to provide expressive and performant tools within the YGM framework. -Interaction with containers occurs in one of two classes of operations: -:code:`for_all` and `async_visit`. - -Both classes expect a function as a primary argument, similar to -:code:`ygm::comm::async`. -However, the passed function signature must match the contents of the container. -Value store containers holding :code:`value_type` objects expect the first -argument of passed functions to address objects with the syntax -:code:`[](value_type &data_item){}`. -Key-value store containers expect these functions instead to support separate -:code:`key_type` (which must be immutable) and :code:`value_type` arguments with -the syntax :code:`[](key_type key, value_type &value){}`. -Although all of these operations agree as to how contained objects are addressed -by functions, the interfaces are subtly different and support additional -optional features. + +Implemented Storage Containers +====================== + +The currently implemented containers include a mix of distributed versions of familiar containers and +distributed-specific containers: + + * ``ygm::container::bag`` - An unordered collection of objects partitioned across processes. Ideally suited for + iteration over all items with no capability for identifying or searching for an individual item within the bag. + * ``ygm::container::set`` - Analogous to ``std::set``. An unordered collection of unique objects with the ability to iterate + and search for individual items. Insertion and iteration are slower than a ``ygm::container::bag``. + * ``ygm::container::multiset`` - Analogous to ``std::multiset``. A set where multiple instances of the same object + may appear. + * ``ygm::container::map`` - Analogous to ``std::map``. A collection of keys with assigned values. Keys and values can + be inserted and looked up individually or iterated over collectively. + * ``ygm::container::multimap`` - Analogous to ``std::multimap``. A map where keys may appear with multiple values. + * ``ygm::container::array`` - A collection of items indexed by an integer type. Items can be inserted and looked up + by their index values independently or iterated over collectively. Differs from a ``std::array`` in that sizes do + not need to known at compile-time, and a ``ygm::container::array`` can be dynamically resized through a + (potentially expensive) function at runtime. + * ``ygm::container::counting_set`` - A container for counting occurrences of items. Can be thought of as a + ``ygm::container::map`` that maps items to integer counts but optimized for the case of frequent duplication of + keys. + * ``ygm::container::disjoint_set`` - A distributed disjoint set data structure. Implements asynchronous union + operation for maintaining membership of items within mathematical disjoint sets. Eschews the find operation of most + disjoint set data structures and instead allows for execution of user-provided lambdas upon successful completion + of set merges. + +Typical Container Operations +============================ + +Most interaction with containers occurs in one of two classes of operations: +:code:`for_all` and `async_`. + +:code:`for_all` Operations +-------------------------- :code:`for_all`-class operations are barrier-inducing collectives that direct -ranks to iteratively apply the passed function to all locally-held data. +ranks to iteratively apply a user-provided function to all locally-held data. Functions passed to the :code:`for_all` interface do not support additional variadic parameters. However, these functions are stored and executed locally on each rank, and so can capture objects in rank-local scope. -:code:`async_visit`-class operations provide a mechanism for executing a -function at a particular piece of data stored within a container. -YGM handles the creation and invocation of a YGM communicator :code:`async` -call, freeing the user to consider algorithmic details. -Not all containers support :code:`async_visit`-class operations. +:code:`async_` Operations +------------------------- + +Operations prefixed with ``async_`` perform operations on containers that can be spawned from any process and +execute on the correct process using YGM's asynchronous runtime. The most common `async` operations are: + + * ``async_insert`` - Inserts an item or a key and value, depending on the container being used. The process responsible + for storing the inserted object is determined using the container's partitioner. Depending on the container, this + partitioner may determine this location using a hash of the item or by heuristics that attempt to evenly spread + data across processes (in the case of ``ygm::container::bag``). + * ``async_visit`` - Items within YGM containers will be distributed across the universe of running processes. Instead + of providing operations to look up this data directly, which would involve a round-trip communication with the + process storing the item of interest, most YGM containers provide ``async_visit``. A call to ``async_visit`` takes + a function to execute and arguments to pass to the function and asynchronously executes the provided function with + arguments that are the item stored in the container and the additional arguments passed to ``async_visit``. + +Specific containers may have additional ``async_`` operations (or may be missing some of the above) based on the +capabilities of the container. Consult the documentation of individual containers for more details. .. toctree:: :maxdepth: 2 @@ -53,7 +87,20 @@ Not all containers support :code:`async_visit`-class operations. container/multiset container/set -YGM also supports adaptor classes and functions that wrap an existing class to -either add or modify operation functionality. +YGM Container Example +===================== + +.. literalinclude:: ../../../examples/container/map_visit.cpp + :language: C++ + +Container Transformation Objects +================================ + +``ygm::container`` provides a number of transformation objects that can be applied to containers to alter the appearance +of items passed to ``for_all`` operations without modifying the items within the container itself. The currently +supported transformation objects are: -.. doxygenfunction:: ygm::container::reduce_by_key_map \ No newline at end of file + * ``filter`` - Filters items in a container to only execute on the portion of the container satisfying a provided + boolean function. + * ``flatten`` - Extract the elements from tuple-like objects before passing to the user's ``for_all`` function. + * ``map`` - Apply a generic function to the container's items before passing to the user's ``for_all`` function. From f84714df8d838639389d8dfc71ad584eaa39039d Mon Sep 17 00:00:00 2001 From: Roger Pearce Date: Fri, 2 Aug 2024 16:23:40 -0500 Subject: [PATCH 10/40] Renamed key_gather to gather_keys. (#231) --- examples/container/word_counter.cpp | 2 +- include/ygm/container/counting_set.hpp | 4 ++-- include/ygm/container/map.hpp | 4 ++-- include/ygm/container/tagged_bag.hpp | 6 +++--- test/test_counting_set.cpp | 6 +++--- test/test_map.cpp | 2 +- test/test_tagged_bag.cpp | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/container/word_counter.cpp b/examples/container/word_counter.cpp index 2da77508..9fecdc50 100644 --- a/examples/container/word_counter.cpp +++ b/examples/container/word_counter.cpp @@ -60,7 +60,7 @@ int main(int argc, char **argv) { to_gather = {"freedom"}; } - auto counts = word_counter.key_gather(to_gather); + auto counts = word_counter.gather_keys(to_gather); for (auto &word_count : counts) { std::cout << word_count.first << " -> " << word_count.second << std::endl; diff --git a/include/ygm/container/counting_set.hpp b/include/ygm/container/counting_set.hpp index f6974b31..b85b219f 100644 --- a/include/ygm/container/counting_set.hpp +++ b/include/ygm/container/counting_set.hpp @@ -133,9 +133,9 @@ class counting_set // return m_map.all_gather(keys); // } - std::map key_gather( + std::map gather_keys( const std::vector &keys) { - return m_map.key_gather(keys); + return m_map.gather_keys(keys); } typename ygm::ygm_ptr get_ygm_ptr() const { return pthis; } diff --git a/include/ygm/container/map.hpp b/include/ygm/container/map.hpp index f0630022..8304bf44 100644 --- a/include/ygm/container/map.hpp +++ b/include/ygm/container/map.hpp @@ -186,7 +186,7 @@ class map } template - std::map key_gather(const STLKeyContainer& keys) { + std::map gather_keys(const STLKeyContainer& keys) { std::map to_return; static std::map& sto_return = to_return; @@ -475,7 +475,7 @@ class multimap } // template - // std::map key_gather(const STLKeyContainer& keys) { + // std::map gather_keys(const STLKeyContainer& keys) { // std::map to_return; // static std::map& sto_return = to_return; diff --git a/include/ygm/container/tagged_bag.hpp b/include/ygm/container/tagged_bag.hpp index 1e2c8bc5..470a8a74 100644 --- a/include/ygm/container/tagged_bag.hpp +++ b/include/ygm/container/tagged_bag.hpp @@ -99,12 +99,12 @@ class tagged_bag { } // template - // std::map key_gather(const STLKeyContainer &tags) { + // std::map gather_keys(const STLKeyContainer &tags) { // return m_tagged_bag.all_gather(tags); // } - std::map key_gather(const std::vector &tags) { - return m_tagged_bag.key_gather(tags); + std::map gather_keys(const std::vector &tags) { + return m_tagged_bag.gather_keys(tags); } template void local_for_all(Function fn) { diff --git a/test/test_counting_set.cpp b/test/test_counting_set.cpp index 19cf6cd1..206a644a 100644 --- a/test/test_counting_set.cpp +++ b/test/test_counting_set.cpp @@ -40,7 +40,7 @@ int main(int argc, char **argv) { YGM_ASSERT_RELEASE(cset.count("red") == 1); YGM_ASSERT_RELEASE(cset.size() == 3); - auto count_map = cset.key_gather({"dog", "cat", "apple"}); + auto count_map = cset.gather_keys({"dog", "cat", "apple"}); YGM_ASSERT_RELEASE(count_map["dog"] == 1); YGM_ASSERT_RELEASE(count_map["apple"] == 1); YGM_ASSERT_RELEASE(count_map.count("cat") == 0); @@ -60,7 +60,7 @@ int main(int argc, char **argv) { YGM_ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); YGM_ASSERT_RELEASE(cset.size() == 3); - auto count_map = cset.key_gather({"dog", "cat", "apple"}); + auto count_map = cset.gather_keys({"dog", "cat", "apple"}); YGM_ASSERT_RELEASE(count_map["dog"] == (size_t)world.size()); YGM_ASSERT_RELEASE(count_map["apple"] == (size_t)world.size()); YGM_ASSERT_RELEASE(cset.count("cat") == 0); @@ -85,7 +85,7 @@ int main(int argc, char **argv) { YGM_ASSERT_RELEASE(cset.count("red") == (size_t)world.size()); YGM_ASSERT_RELEASE(cset.size() == 3); - auto count_map = cset.key_gather({"dog", "cat", "apple"}); + auto count_map = cset.gather_keys({"dog", "cat", "apple"}); YGM_ASSERT_RELEASE(count_map["dog"] == (size_t)world.size()); YGM_ASSERT_RELEASE(count_map["apple"] == (size_t)world.size()); YGM_ASSERT_RELEASE(cset.count("cat") == 0); diff --git a/test/test_map.cpp b/test/test_map.cpp index 821e3a3a..7f812a51 100644 --- a/test/test_map.cpp +++ b/test/test_map.cpp @@ -236,7 +236,7 @@ int main(int argc, char **argv) { gather_list.clear(); } - auto gmap = smap.key_gather(gather_list); + auto gmap = smap.gather_keys(gather_list); if (world.rank0()) { YGM_ASSERT_RELEASE(gmap["foo"][0] == "bar"); diff --git a/test/test_tagged_bag.cpp b/test/test_tagged_bag.cpp index 3c5b8138..4f7963e5 100644 --- a/test/test_tagged_bag.cpp +++ b/test/test_tagged_bag.cpp @@ -32,7 +32,7 @@ int main(int argc, char** argv) { YGM_ASSERT_RELEASE(tagbag.size() == 3); // Test gather - auto gather = tagbag.key_gather(r0tags); + auto gather = tagbag.gather_keys(r0tags); world.barrier(); if (world.rank0()) { YGM_ASSERT_RELEASE(gather.size() == 3); @@ -42,7 +42,7 @@ int main(int argc, char** argv) { tagbag.for_all([](auto& k, auto& v) { v += "_added"; }); - auto gatheradd = tagbag.key_gather(r0tags); + auto gatheradd = tagbag.gather_keys(r0tags); if (world.rank0()) { for (auto r0tag : r0tags) { auto ga = gatheradd.at(r0tag); From 32fcffaa82e1c0984ea70b3e0ce3ece3eb71652d Mon Sep 17 00:00:00 2001 From: Trevor Steil Date: Fri, 2 Aug 2024 20:38:06 -0700 Subject: [PATCH 11/40] Adds batch erase functionality to ygm::container::multiset (#232) --- .../ygm/container/detail/base_batch_erase.hpp | 30 ++++++++ include/ygm/container/set.hpp | 29 +++---- test/test_multiset.cpp | 55 +++++++++++--- test/test_set.cpp | 75 +++++++++++++------ 4 files changed, 144 insertions(+), 45 deletions(-) create mode 100644 include/ygm/container/detail/base_batch_erase.hpp diff --git a/include/ygm/container/detail/base_batch_erase.hpp b/include/ygm/container/detail/base_batch_erase.hpp new file mode 100644 index 00000000..a359cb60 --- /dev/null +++ b/include/ygm/container/detail/base_batch_erase.hpp @@ -0,0 +1,30 @@ +// Copyright 2019-2021 Lawrence Livermore National Security, LLC and other YGM +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +namespace ygm::container::detail { + +template +struct base_batch_erase_key { + template + void erase(const Container &cont) requires detail::HasForAll && + detail::SingleItemTuple && + detail::AtLeastOneItemTuple && std::convertible_to< + std::tuple_element_t<0, typename Container::for_all_args>, + std::tuple_element_t<0, for_all_args>> { + derived_type *derived_this = static_cast(this); + + cont.for_all( + [derived_this](const auto &key) { derived_this->async_erase(key); }); + + derived_this->comm().barrier(); + } +}; +} // namespace ygm::container::detail diff --git a/include/ygm/container/set.hpp b/include/ygm/container/set.hpp index cddf9d0c..e02088ba 100644 --- a/include/ygm/container/set.hpp +++ b/include/ygm/container/set.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ class multiset : public detail::base_async_insert_value, std::tuple>, public detail::base_async_erase_key, std::tuple>, + public detail::base_batch_erase_key, std::tuple>, public detail::base_async_contains, std::tuple>, public detail::base_async_insert_contains, std::tuple>, @@ -71,9 +73,9 @@ class multiset } template - multiset(ygm::comm &comm, const STLContainer &cont) - requires detail::STLContainer && - std::convertible_to + multiset(ygm::comm &comm, const STLContainer &cont) requires + detail::STLContainer && + std::convertible_to : m_comm(comm), pthis(this), partitioner(comm) { pthis.check(m_comm); @@ -85,10 +87,9 @@ class multiset } template - multiset(ygm::comm &comm, const YGMContainer &yc) - requires detail::HasForAll && - detail::SingleItemTuple< - typename YGMContainer::for_all_args> //&& + multiset(ygm::comm &comm, + const YGMContainer &yc) requires detail::HasForAll && + detail::SingleItemTuple //&& // std::same_as> : m_comm(comm), pthis(this), partitioner(comm) { pthis.check(m_comm); @@ -151,6 +152,7 @@ template class set : public detail::base_async_insert_value, std::tuple>, public detail::base_async_erase_key, std::tuple>, + public detail::base_batch_erase_key, std::tuple>, public detail::base_async_contains, std::tuple>, public detail::base_async_insert_contains, std::tuple>, public detail::base_count, std::tuple>, @@ -197,9 +199,9 @@ class set } template - set(ygm::comm &comm, const STLContainer &cont) - requires detail::STLContainer && - std::convertible_to + set(ygm::comm &comm, + const STLContainer &cont) requires detail::STLContainer && + std::convertible_to : m_comm(comm), pthis(this), partitioner(comm) { pthis.check(m_comm); @@ -210,10 +212,9 @@ class set } template - set(ygm::comm &comm, const YGMContainer &yc) - requires detail::HasForAll && - detail::SingleItemTuple< - typename YGMContainer::for_all_args> //&& + set(ygm::comm &comm, + const YGMContainer &yc) requires detail::HasForAll && + detail::SingleItemTuple //&& // std::same_as> : m_comm(comm), pthis(this), partitioner(comm) { pthis.check(m_comm); diff --git a/test/test_multiset.cpp b/test/test_multiset.cpp index f04b315c..ef06b961 100644 --- a/test/test_multiset.cpp +++ b/test/test_multiset.cpp @@ -58,14 +58,12 @@ int main(int argc, char** argv) { // // Test async_contains { - static bool set_contains = false; + static bool set_contains = false; ygm::container::multiset iset(world); world.barrier(); int val = 42; - auto f = [](bool contains, const int& i) { - set_contains = contains; - }; + auto f = [](bool contains, const int& i) { set_contains = contains; }; if (world.rank0()) { iset.async_contains(val, f); @@ -87,13 +85,13 @@ int main(int argc, char** argv) { // // Test async_insert_contains { - static bool already_contains = false; + static bool already_contains = false; ygm::container::multiset sset(world); world.barrier(); auto f = [](bool& contains, const std::string& s) { already_contains = contains; - }; + }; if (world.rank0()) { sset.async_insert_contains("dog", f); @@ -108,6 +106,45 @@ int main(int argc, char** argv) { YGM_ASSERT_RELEASE(ygm::logical_or(already_contains, world)); } + // Test batch erase + { + int num_items = 100; + int num_insertion_rounds = 5; + int remove_size = 20; + ygm::container::multiset iset(world); + + if (world.rank0()) { + for (int round = 0; round < num_insertion_rounds; ++round) { + for (int i = 0; i < num_items; ++i) { + iset.async_insert(i); + } + } + } + + world.barrier(); + + YGM_ASSERT_RELEASE(iset.size() == num_insertion_rounds * num_items); + + ygm::container::set to_remove(world); + + if (world.rank0()) { + for (int i = 0; i < remove_size; ++i) { + to_remove.async_insert(i); + } + } + + world.barrier(); + + iset.erase(to_remove); + + iset.for_all([remove_size, &world](const auto& item) { + YGM_ASSERT_RELEASE(item >= remove_size); + }); + + YGM_ASSERT_RELEASE(iset.size() == + num_insertion_rounds * (num_items - remove_size)); + } + // // Test swap { @@ -139,7 +176,7 @@ int main(int argc, char** argv) { sset1.async_insert("apple"); sset1.async_insert("red"); - sset1.for_all([&sset2](const auto &key) { sset2.async_insert(key); }); + sset1.for_all([&sset2](const auto& key) { sset2.async_insert(key); }); YGM_ASSERT_RELEASE(sset2.count("dog") == world.size()); YGM_ASSERT_RELEASE(sset2.count("apple") == world.size()); @@ -149,7 +186,7 @@ int main(int argc, char** argv) { // // Test vector of sets { - int num_sets = 4; + int num_sets = 4; std::vector> vec_sets; for (int i = 0; i < num_sets; ++i) { @@ -169,4 +206,4 @@ int main(int argc, char** argv) { } return 0; -} \ No newline at end of file +} diff --git a/test/test_set.cpp b/test/test_set.cpp index 17d4f4e1..b2b7cea8 100644 --- a/test/test_set.cpp +++ b/test/test_set.cpp @@ -9,10 +9,10 @@ #include #include -#include #include +#include -int main(int argc, char **argv) { +int main(int argc, char** argv) { ygm::comm world(&argc, &argv); // @@ -29,7 +29,7 @@ int main(int argc, char **argv) { // // Test Rank 0 async_insert - { + { ygm::container::set sset(world); if (world.rank() == 0) { sset.async_insert("dog"); @@ -57,7 +57,7 @@ int main(int argc, char **argv) { // Test Rank 0 async_insert with ygm set pointer { ygm::container::set sset(world); - auto sset_ptr = sset.get_ygm_ptr(); + auto sset_ptr = sset.get_ygm_ptr(); if (world.rank() == 0) { sset_ptr->async_insert("dog"); sset_ptr->async_insert("apple"); @@ -69,7 +69,6 @@ int main(int argc, char **argv) { YGM_ASSERT_RELEASE(sset.size() == 3); } - // // Test all ranks async_insert { @@ -88,17 +87,14 @@ int main(int argc, char **argv) { YGM_ASSERT_RELEASE(sset.size() == 2); } - // // Test async_contains { static bool set_contains = false; ygm::container::set iset(world); - int val = 42; + int val = 42; - auto f = [](bool& contains, const int& i) { - set_contains = contains; - }; + auto f = [](bool& contains, const int& i) { set_contains = contains; }; if (world.rank0()) { iset.async_contains(val, f); @@ -121,12 +117,12 @@ int main(int argc, char **argv) { // // Test async_insert_contains { - static bool did_contain = false; + static bool did_contain = false; ygm::container::set sset(world); auto f = [](bool& contains, const std::string& s) { did_contain = contains; - }; + }; if (world.rank0()) { sset.async_insert_contains("dog", f); @@ -141,10 +137,45 @@ int main(int argc, char **argv) { YGM_ASSERT_RELEASE(ygm::logical_or(did_contain, world)); } + // Test batch erase + { + int num_items = 100; + int remove_size = 20; + ygm::container::set iset(world); + + if (world.rank0()) { + for (int i = 0; i < num_items; ++i) { + iset.async_insert(i); + } + } + + world.barrier(); + + YGM_ASSERT_RELEASE(iset.size() == num_items); + + ygm::container::set to_remove(world); + + if (world.rank0()) { + for (int i = 0; i < remove_size; ++i) { + to_remove.async_insert(i); + } + } + + world.barrier(); + + iset.erase(to_remove); + + iset.for_all([remove_size, &world](const auto& item) { + YGM_ASSERT_RELEASE(item >= remove_size); + }); + + YGM_ASSERT_RELEASE(iset.size() == num_items - remove_size); + } // Test from bag { - ygm::container::bag sbag(world, {"one", "two", "three", "one", "two"}); + ygm::container::bag sbag( + world, {"one", "two", "three", "one", "two"}); YGM_ASSERT_RELEASE(sbag.size() == 5); ygm::container::set sset(world, sbag); @@ -153,28 +184,27 @@ int main(int argc, char **argv) { // Test initializer list { - ygm::container::set sset(world, {"one", "two", "three", "one", "two"}); + ygm::container::set sset( + world, {"one", "two", "three", "one", "two"}); YGM_ASSERT_RELEASE(sset.size() == 3); } // Test from STL vector { - std::vector v({1,2,3,4,5,1,1,1,3}); + std::vector v({1, 2, 3, 4, 5, 1, 1, 1, 3}); ygm::container::set iset(world, v); YGM_ASSERT_RELEASE(iset.size() == 5); } - // // Test additional arguments of async_contains // { // ygm::container::set sset(world); - // sset.async_contains("howdy", [](bool c, const std::string s, int i, float f){}, 3, 3.14); - // sset.async_contains("howdy", [](auto ptr_set, bool c, const std::string s){}); - // world.barrier(); + // sset.async_contains("howdy", [](bool c, const std::string s, int i, float + // f){}, 3, 3.14); sset.async_contains("howdy", [](auto ptr_set, bool c, + // const std::string s){}); world.barrier(); // } - // // Test swap { @@ -206,7 +236,7 @@ int main(int argc, char **argv) { sset1.async_insert("apple"); sset1.async_insert("red"); - sset1.for_all([&sset2](const auto &key) { sset2.async_insert(key); }); + sset1.for_all([&sset2](const auto& key) { sset2.async_insert(key); }); YGM_ASSERT_RELEASE(sset2.count("dog") == 1); YGM_ASSERT_RELEASE(sset2.count("apple") == 1); @@ -223,7 +253,8 @@ int main(int argc, char **argv) { // sset1.async_insert("apple"); // sset1.async_insert("red"); - // sset1.consume_all([&sset2](const auto &key) { sset2.async_insert(key); }); + // sset1.consume_all([&sset2](const auto &key) { sset2.async_insert(key); + // }); // YGM_ASSERT_RELEASE(sset1.empty()); // YGM_ASSERT_RELEASE(sset2.count("dog") == 1); From 2b6ae17e0db6261f7559fabcb33a454139c80fcc Mon Sep 17 00:00:00 2001 From: Roger Pearce Date: Sat, 3 Aug 2024 14:22:06 -0500 Subject: [PATCH 12/40] Added gather_topk, separated base_iteration. (#233) --- include/ygm/container/counting_set.hpp | 5 + .../ygm/container/detail/base_iteration.hpp | 259 ++++++++++++++++-- include/ygm/container/detail/map_proxy.hpp | 33 +-- include/ygm/container/detail/type_traits.hpp | 48 ++++ include/ygm/container/map.hpp | 6 +- test/CMakeLists.txt | 1 + test/test_gather_topk.cpp | 46 ++++ 7 files changed, 342 insertions(+), 56 deletions(-) create mode 100644 include/ygm/container/detail/type_traits.hpp create mode 100644 test/test_gather_topk.cpp diff --git a/include/ygm/container/counting_set.hpp b/include/ygm/container/counting_set.hpp index b85b219f..acd7ac8f 100644 --- a/include/ygm/container/counting_set.hpp +++ b/include/ygm/container/counting_set.hpp @@ -90,6 +90,11 @@ class counting_set m_map.local_for_all(fn); } + template + void local_for_all(Function fn) const { + m_map.local_for_all(fn); + } + void local_clear() { // What to do here m_map.local_clear(); clear_cache(); diff --git a/include/ygm/container/detail/base_iteration.hpp b/include/ygm/container/detail/base_iteration.hpp index 76456529..97b63641 100644 --- a/include/ygm/container/detail/base_iteration.hpp +++ b/include/ygm/container/detail/base_iteration.hpp @@ -6,6 +6,7 @@ #pragma once #include +#include #include #include @@ -22,6 +23,14 @@ class flatten_proxy; template struct base_iteration { + static_assert(sizeof(for_all_args) != sizeof(for_all_args), + "Unsupported for_all_args"); +}; + +template +struct base_iteration { + using value_type = typename std::tuple_element<0, for_all_args>::type; + template void for_all(Function fn) { derived_type* derived_this = static_cast(this); @@ -38,6 +47,8 @@ struct base_iteration { template void gather(STLContainer& gto, int rank) const { + static_assert( + std::is_same_v); // TODO, make an all gather version that defaults to rank = -1 & uses a temp // container. bool all_gather = (rank == -1); @@ -56,14 +67,47 @@ struct base_iteration { derived_this->comm().barrier(); } - template - std::tuple_element<0, for_all_args>::type reduce(MergeFunction merge) const + template > + std::vector gather_topk( + size_t k, Compare comp = std::greater()) const requires SingleItemTuple { + const derived_type* derived_this = static_cast(this); + const ygm::comm& mycomm = derived_this->comm(); + std::vector local_topk; + + // + // Find local top_k + for_all([&local_topk, comp, k](const value_type& value) { + local_topk.push_back(value); + std::sort(local_topk.begin(), local_topk.end(), comp); + if (local_topk.size() > k) { + local_topk.pop_back(); + } + }); + + // + // All reduce global top_k + auto to_return = mycomm.all_reduce( + local_topk, [comp, k](const std::vector& va, + const std::vector& vb) { + std::vector out(va.begin(), va.end()); + out.insert(out.end(), vb.begin(), vb.end()); + std::sort(out.begin(), out.end(), comp); + while (out.size() > k) { + out.pop_back(); + } + return out; + }); + return to_return; + } + + template + value_type reduce(MergeFunction merge) const { const derived_type* derived_this = static_cast(this); derived_this->comm().barrier(); YGM_ASSERT_RELEASE(derived_this->local_size() > - 0); // empty partition not handled yet + 0); // empty partition not handled yet using value_type = typename std::tuple_element<0, for_all_args>::type; bool first = true; @@ -89,8 +133,162 @@ struct base_iteration { template void collect(YGMContainer& c) const { const derived_type* derived_this = static_cast(this); - auto clambda = [&c](const std::tuple_element<0, for_all_args>::type& item) { - c.async_insert(item); + auto clambda = [&c](const value_type& item) { c.async_insert(item); }; + derived_this->for_all(clambda); + } + + template + void reduce_by_key(MapType& map, ReductionOp reducer) const { + // TODO: static_assert MapType is ygm::container::map + const derived_type* derived_this = static_cast(this); + using reduce_key_type = typename MapType::key_type; + using reduce_value_type = typename MapType::mapped_type; + static_assert(std::is_same_v>, + "value_type must be a std::pair"); + + auto rbklambda = + [&map, reducer](std::pair kvp) { + map.async_reduce(kvp.first, kvp.second, reducer); + }; + derived_this->for_all(rbklambda); + } + + template + map_proxy map(MapFunction ffn); + + flatten_proxy flatten(); + + template + filter_proxy filter(FilterFunction ffn); + + private: + template + requires requires(STLContainer stc, Value v) { stc.push_back(v); } + static void generic_insert(STLContainer& stc, const Value& value) { + stc.push_back(value); + } + + template + requires requires(STLContainer stc, Value v) { stc.insert(v); } + static void generic_insert(STLContainer& stc, const Value& value) { + stc.insert(value); + } +}; + +template +struct base_iteration { + using key_type = typename std::tuple_element<0, for_all_args>::type; + using mapped_type = typename std::tuple_element<1, for_all_args>::type; + + template + void for_all(Function fn) { + derived_type* derived_this = static_cast(this); + derived_this->comm().barrier(); + derived_this->local_for_all(fn); + } + + template + void for_all(Function fn) const { + const derived_type* derived_this = static_cast(this); + derived_this->comm().barrier(); + derived_this->local_for_all(fn); + } + + template + void gather(STLContainer& gto, int rank) const { + static_assert(std::is_same_v>); + // TODO, make an all gather version that defaults to rank = -1 & uses a temp + // container. + bool all_gather = (rank == -1); + static STLContainer* spgto = >o; + const derived_type* derived_this = static_cast(this); + const ygm::comm& mycomm = derived_this->comm(); + + auto glambda = [&mycomm, rank](const key_type& key, + const mapped_type& value) { + mycomm.async( + rank, + [](const key_type& key, const mapped_type& value) { + generic_insert(*spgto, std::make_pair(key, value)); + }, + key, value); + }; + + for_all(glambda); + + derived_this->comm().barrier(); + } + + template >> + std::vector> gather_topk( + size_t k, Compare comp = Compare()) const { + const derived_type* derived_this = static_cast(this); + const ygm::comm& mycomm = derived_this->comm(); + using vec_type = std::vector>; + vec_type local_topk; + + // + // Find local top_k + for_all( + [&local_topk, comp, k](const key_type& key, const mapped_type& mapped) { + local_topk.push_back(std::make_pair(key, mapped)); + std::sort(local_topk.begin(), local_topk.end(), comp); + if (local_topk.size() > k) { + local_topk.pop_back(); + } + }); + + // + // All reduce global top_k + auto to_return = mycomm.all_reduce( + local_topk, [comp, k](const vec_type& va, const vec_type& vb) { + vec_type out(va.begin(), va.end()); + out.insert(out.end(), vb.begin(), vb.end()); + std::sort(out.begin(), out.end(), comp); + while (out.size() > k) { + out.pop_back(); + } + return out; + }); + return to_return; + } + + template + std::pair reduce(MergeFunction merge) const { + const derived_type* derived_this = static_cast(this); + derived_this->comm().barrier(); + + bool first = true; + + std::pair local_reduce; + + auto rlambda = [&local_reduce, &first, + &merge](const std::pair& value) { + if (first) { + local_reduce = value; + first = false; + } else { + local_reduce = merge(local_reduce, value); + } + }; + + derived_this->for_all(rlambda); + + std::optional> to_reduce; + if (first) { // local partition was empty! + to_reduce = std::move(local_reduce); + } + + return ::ygm::all_reduce(to_reduce, merge, derived_this->comm()); + } + + template + void collect(YGMContainer& c) const { + const derived_type* derived_this = static_cast(this); + auto clambda = [&c](const key_type& key, const mapped_type& value) { + c.async_insert(std::make_pair(key, value)); }; derived_this->for_all(clambda); } @@ -101,19 +299,13 @@ struct base_iteration { // static_assert ygm::map using reduce_key_type = typename MapType::key_type; using reduce_value_type = typename MapType::mapped_type; - if constexpr (std::tuple_size::value == 1) { - // must be a std::pair - auto rbklambda = [&map, reducer](std::pair kvp) { - map.async_reduce(kvp.first, kvp.second, reducer); - }; - derived_this->for_all(rbklambda); - } else { - static_assert(std::tuple_size::value == 2); - auto rbklambda = [&map, reducer](const reduce_key_type& key, const reduce_value_type& value) { - map.async_reduce(key, value, reducer); - }; - derived_this->for_all(rbklambda); - } + + static_assert(std::tuple_size::value == 2); + auto rbklambda = [&map, reducer](const reduce_key_type& key, + const reduce_value_type& value) { + map.async_reduce(key, value, reducer); + }; + derived_this->for_all(rbklambda); } template @@ -146,7 +338,7 @@ struct base_iteration { namespace ygm::container::detail { -template +template template map_proxy base_iteration::map(MapFunction ffn) { @@ -154,7 +346,7 @@ base_iteration::map(MapFunction ffn) { return map_proxy(*derived_this, ffn); } -template +template inline flatten_proxy base_iteration::flatten() { // static_assert( @@ -163,7 +355,32 @@ base_iteration::flatten() { return flatten_proxy(*derived_this); } -template +template +template +filter_proxy +base_iteration::filter(FilterFunction ffn) { + derived_type* derived_this = static_cast(this); + return filter_proxy(*derived_this, ffn); +} + +template +template +map_proxy +base_iteration::map(MapFunction ffn) { + derived_type* derived_this = static_cast(this); + return map_proxy(*derived_this, ffn); +} + +template +inline flatten_proxy +base_iteration::flatten() { + // static_assert( + // type_traits::is_vector>::value); + derived_type* derived_this = static_cast(this); + return flatten_proxy(*derived_this); +} + +template template filter_proxy base_iteration::filter(FilterFunction ffn) { diff --git a/include/ygm/container/detail/map_proxy.hpp b/include/ygm/container/detail/map_proxy.hpp index 2dcab046..81a2db59 100644 --- a/include/ygm/container/detail/map_proxy.hpp +++ b/include/ygm/container/detail/map_proxy.hpp @@ -8,42 +8,11 @@ #include #include #include +#include namespace ygm::container::detail { -namespace type_traits { -template