Merge pull request #1 from LLNL/master

Preparing v0.6 Release (LLNL#200)

.github/workflows/ci-test.yaml

@@ -2,7 +2,7 @@ name: CI Test
 
 on:
   pull_request:
-    branches: [ main, develop ]
+    branches: [ master, develop ]
   push:
     branches: [ 'feature/**', 'hotfix/**']
 
@@ -39,9 +39,9 @@ jobs:
         run: |
           cd ~
           wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-          sudo apt install ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
-          sudo apt update
-          sudo apt install libarrow-dev libparquet-dev
+          sudo apt-get install ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt-get update
+          sudo apt-get install libarrow-dev libparquet-dev
       - name: Install mpich
         if: matrix.mpi-type == 'mpich'
         run: sudo apt-get install mpich
@@ -58,8 +58,8 @@ jobs:
           g++-${{ matrix.gcc-version }} --version
           mkdir build
           cd build
-          cmake ../ -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_COMPILER=g++-${{ matrix.gcc-version }} -DBOOST_ROOT=~/boost_1_77_0
-          make
+          cmake ../ -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_COMPILER=g++-${{ matrix.gcc-version }} -DBOOST_ROOT=~/boost_1_77_0 -DYGM_REQUIRE_ARROW=ON
+          make -j
       - name: Make test (mpich)
         if: matrix.mpi-type == 'mpich'
         run: |

CMakeLists.txt

@@ -7,7 +7,7 @@ cmake_minimum_required(VERSION 3.14)
 
 project(
     ygm
-    VERSION 0.5
+    VERSION 0.6
     DESCRIPTION "HPC Communication Library"
     LANGUAGES CXX
 )
@@ -141,6 +141,21 @@ endif()
 if (NOT Arrow_FOUND)
 	find_package(Arrow 10.0 QUIET)
 endif()
+if (NOT Arrow_FOUND)
+	find_package(Arrow 11.0 QUIET)
+endif()
+if (NOT Arrow_FOUND)
+	find_package(Arrow 12.0 QUIET)
+endif()
+if (NOT Arrow_FOUND)
+	find_package(Arrow 13.0 QUIET)
+endif()
+if (NOT Arrow_FOUND)
+	find_package(Arrow 14.0 QUIET)
+endif()
+if (NOT Arrow_FOUND)
+	find_package(Arrow 15.0 QUIET)
+endif()
 if (Arrow_FOUND)
    message(STATUS ${PROJECT_NAME} " found Arrow ")
    message(STATUS "Arrow version: ${ARROW_VERSION}")
@@ -150,12 +165,15 @@ if (Arrow_FOUND)
    if (Parquet_FOUND)
        message(STATUS ${PROJECT_NAME} " found Parquet ")
        message(STATUS "Parquet version: ${PARQUET_VERSION}")
-       message(STATUS "Parquet SO version: ${PARQUET_FULL_SO_VERSION}") 
+       message(STATUS "Parquet SO version: ${PARQUET_FULL_SO_VERSION}")
    else ()
-       message(WARNING ${PROJECT_NAME} " did not find Parquet. Building without Parquet.") 
-   endif () 
+       message(WARNING ${PROJECT_NAME} " did not find Parquet. Building without Parquet.")
+   endif ()
 else ()
-   message(WARNING ${PROJECT_NAME} " did not find Arrow >= 8.0. Building without Arrow.") 
+   message(WARNING ${PROJECT_NAME} " did not find Arrow >= 8.0. Building without Arrow.")
+   if (YGM_REQUIRE_ARROW)
+	   message(FATAL_ERROR "YGM configured to require Arrow, but Arrow could not be found")
+   endif ()
 endif ()
 
 #
@@ -235,7 +253,6 @@ endif ()
 # Testing & examples are only available if this is the main app
 if (YGM_MAIN_PROJECT)
     add_subdirectory(test)
-    add_subdirectory(performance)
     # Example codes are here.
     add_subdirectory(examples)
 endif ()

examples/container/alg_pagerank.cpp

@@ -34,8 +34,8 @@ int main(int argc, char **argv) {
     value = value + update_val;
   };
 
-  auto deg_acc_lambda = [](auto &rv_pair, const auto &update_val) {
-    rv_pair.second = rv_pair.second + update_val;
+  auto deg_acc_lambda = [](auto &row, auto &val, const auto &update_val) {
+    val = val + update_val;
   };
 
   std::string key1, key2;
@@ -59,12 +59,12 @@ int main(int argc, char **argv) {
 
   int N              = pr.size();
   init_pr            = ((double)1) / N;
-  auto mod_pr_lambda = [&init_pr](auto &rv_pair) { rv_pair.second = init_pr; };
+  auto mod_pr_lambda = [&init_pr](const auto &vtx, auto &pg_rnk) {
+    pg_rnk = init_pr;
+  };
   pr.for_all(mod_pr_lambda);
 
-  auto deg_lambda = [&A](const auto &kv_pair) {
-    auto vtx            = kv_pair.first;
-    auto deg            = kv_pair.second;
+  auto deg_lambda = [&A](const auto &vtx, const auto &deg) {
     auto scale_A_lambda = [](const auto &row, const auto &col, auto &value,
                              const auto &deg) {
       value = ((double)value) / deg;
@@ -83,20 +83,18 @@ int main(int argc, char **argv) {
         ns_spmv::spmv(A, pr, std::plus<double>(), std::multiplies<double>());
     world.barrier();
 
-    auto adding_damping_pr_lambda = [&map_res, d_val, N](auto &vtx_pr) {
-      auto vtx_id       = vtx_pr.first;
-      auto pg_rnk       = vtx_pr.second;
-      auto visit_lambda = [](auto &vtx_pr_pair, auto &da_val, auto &d_val) {
-        vtx_pr_pair.second = da_val + d_val * vtx_pr_pair.second;
-      };
-      map_res.async_insert_if_missing_else_visit(vtx_id, (float(1 - d_val) / N),
+    auto adding_damping_pr_lambda = [&map_res, d_val, N](const auto &vtx,
+                                                         const auto &pg_rnk) {
+      auto visit_lambda = [](const auto &vtx_id, auto &pr, auto &da_val,
+                             auto &d_val) { pr = da_val + d_val * pr; };
+      map_res.async_insert_if_missing_else_visit(vtx, (float(1 - d_val) / N),
                                                  visit_lambda, d_val);
     };
     pr.for_all(adding_damping_pr_lambda);
     pr.swap(map_res);
 
-    auto agg_pr_lambda = [&agg_pr](auto &vtx_pr_pair) {
-      agg_pr = agg_pr + vtx_pr_pair.second;
+    auto agg_pr_lambda = [&agg_pr](const auto &vtx, const auto &pg_rnk) {
+      agg_pr = agg_pr + pg_rnk;
     };
     pr.for_all(agg_pr_lambda);
     world.barrier();

examples/container/disjoint_set_cc.cpp

@@ -37,8 +37,7 @@ int main(int argc, char** argv) {
   connected_components.all_compress();
 
   world.cout0("Person : Representative");
-  connected_components.for_all([&world](const auto& person_rep_pair) {
-    std::cout << person_rep_pair.first << " : " << person_rep_pair.second
-              << std::endl;
+  connected_components.for_all([&world](const auto& person, const auto& rep) {
+    std::cout << person << " : " << rep << std::endl;
   });
 }

examples/container/map_insert_if_missing.cpp

@@ -22,10 +22,10 @@ int main(int argc, char **argv) {
 
   world.barrier();
 
-  auto sounds_lambda = [](auto &kv_pair, const auto &new_value,
-                          const int origin_rank) {
-    std::cout << "The " << kv_pair.first << " says " << kv_pair.second
-              << " for rank " << origin_rank << std::endl;
+  auto sounds_lambda = [](const auto &key, const auto &value,
+                          const auto &new_value, const int origin_rank) {
+    std::cout << "The " << key << " says " << value << " for rank "
+              << origin_rank << std::endl;
   };
 
   // Keys already exist. Visits occur instead.

examples/container/map_set.cpp

@@ -7,7 +7,7 @@
 #include <ygm/container/map.hpp>
 #include <ygm/container/set.hpp>
 
-int main(int argc, char** argv) {
+int main(int argc, char **argv) {
   ygm::comm world(&argc, &argv);
 
   ygm::container::set<std::string>              str_set(world);
@@ -27,8 +27,8 @@ int main(int argc, char** argv) {
 
   str_set.for_all([](auto k) { std::cout << "str_set: " << k << std::endl; });
 
-  str_map.for_all([](auto kv) {
-    std::cout << "str_map: " << kv.first << " -> " << kv.second << std::endl;
+  str_map.for_all([](const auto &key, auto &value) {
+    std::cout << "str_map: " << key << " -> " << value << std::endl;
   });
   return 0;
 }

examples/container/map_visit.cpp

@@ -18,10 +18,9 @@ int main(int argc, char **argv) {
 
   world.barrier();
 
-  auto favorites_lambda = [](auto kv_pair, const int favorite_num) {
-    std::cout << "My favorite animal is a " << kv_pair.first << ". It says '"
-              << kv_pair.second << "!' My favorite number is " << favorite_num
-              << std::endl;
+  auto favorites_lambda = [](auto key, auto &value, const int favorite_num) {
+    std::cout << "My favorite animal is a " << key << ". It says '" << value
+              << "!' My favorite number is " << favorite_num << std::endl;
   };
 
   // Send visitors to map

examples/container/map_visit_optional_arguments.cpp

@@ -17,20 +17,19 @@ int main(int argc, char **argv) {
 
   world.barrier();
 
-  auto visit_lambda = [](auto pmap, auto kv_pair) {
+  auto visit_lambda = [](auto pmap, auto key, auto value) {
     std::cout << "Rank " << pmap->comm().rank() << " is receiving a lookup\n"
-              << "\tKey: " << kv_pair.first << " Value: " << kv_pair.second
+              << "\tKey: " << key << " Value: " << value
               << "\n\tGoing to ask rank 0 to say something." << std::endl;
 
     // Send message to rank 0 to introduce himself
-    pmap->comm().async(0,
-                       [](auto pcomm, int from) {
-                         std::cout << "Hi. I'm rank " << pcomm->rank()
-                                   << ". Rank " << from
-                                   << " wanted me to say something."
-                                   << std::endl;
-                       },
-                       pmap->comm().rank());
+    pmap->comm().async(
+        0,
+        [](auto pcomm, int from) {
+          std::cout << "Hi. I'm rank " << pcomm->rank() << ". Rank " << from
+                    << " wanted me to say something." << std::endl;
+        },
+        pmap->comm().rank());
   };
 
   // Send lookup from odd-numbered ranks

examples/container/multimap_visit_group.cpp

@@ -21,9 +21,8 @@ int main(int argc, char **argv) {
   world.cout0("Visiting individual key-value pairs with async_visit");
 
   // async_visit gives access to individual key-value pairs
-  auto visit_lambda = [](auto kv_pair) {
-    std::cout << "One thing a " << kv_pair.first << " says is "
-              << kv_pair.second << std::endl;
+  auto visit_lambda = [](const auto &key, const auto &value) {
+    std::cout << "One thing a " << key << " says is " << value << std::endl;
   };
 
   if (world.rank() % 2) {

examples/io/CMakeLists.txt

@@ -1,10 +1,21 @@
-# Copyright 2019-2021 Lawrence Livermore National Security, LLC and other YGM
+# Copyright 2019-2023 Lawrence Livermore National Security, LLC and other YGM
 # Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: MIT
 
 if (Arrow_FOUND AND Parquet_FOUND)
     add_ygm_example(arrow_parquet_stream_reader)
-    target_link_libraries(arrow_parquet_stream_reader PUBLIC arrow_shared parquet_shared)
-endif()
+    target_link_libraries(arrow_parquet_stream_reader PUBLIC
+        Arrow::arrow_shared Parquet::parquet_shared)
 
+    add_ygm_example(arrow_parquet_stream_reader_variant)
+    target_link_libraries(arrow_parquet_stream_reader_variant PUBLIC
+            Arrow::arrow_shared Parquet::parquet_shared)
+
+    if (Boost_FOUND)
+        add_ygm_example(arrow_parquet_stream_reader_json)
+        target_include_directories(arrow_parquet_stream_reader_json PUBLIC ${Boost_INCLUDE_DIRS})
+        target_link_libraries(arrow_parquet_stream_reader_json PUBLIC
+            Arrow::arrow_shared Parquet::parquet_shared)
+    endif()
+endif()

examples/io/arrow_parquet_stream_reader_json.cpp

@@ -0,0 +1,53 @@
+// Copyright 2019-2023 Lawrence Livermore National Security, LLC and other YGM
+// Project Developers. See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: MIT
+
+// Usage:
+// cd /ygm/build/dir
+// mpirun -np 2 ./arrow_parquet_stream_reader_json \
+//  [(option) /path/to/parquet/file/or/dir]
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include <boost/json/src.hpp>
+
+#include <ygm/comm.hpp>
+#include <ygm/detail/cereal_boost_json.hpp>
+#include <ygm/io/detail/arrow_parquet_json_converter.hpp>
+
+int main(int argc, char** argv) {
+  ygm::comm world(&argc, &argv);
+
+  world.cout0()
+      << "Arrow Parquet file parser example (reads data as JSON objects)"
+      << std::endl;
+
+  // assuming the build directory is inside the YGM root directory
+  std::string dir_name = "../test/data/parquet_files_json/";
+  if (argc == 2) {
+    dir_name = argv[1];
+  }
+
+  ygm::io::arrow_parquet_parser parquetp(world, {dir_name});
+
+  world.cout0() << "Schema:\n" << parquetp.schema_to_string() << std::endl;
+
+  world.cout0() << "Read data as JSON:" << std::endl;
+  const auto& schema = parquetp.schema();
+  parquetp.for_all([&schema, &world](auto& stream_reader, const auto&) {
+    // obj's type is boost::json::object
+    const auto obj =
+        ygm::io::detail::read_parquet_as_json(stream_reader, schema);
+
+    world.async(
+        0, [](auto, const auto& obj) { std::cout << obj << std::endl; }, obj);
+  });
+
+  return 0;
+}