Merge pull request #257 from LLNL/release/v0.23

Release/v0.23

.github/workflows/ci-test.yml

@@ -3,7 +3,7 @@ name: CI Test
 on: [push, pull_request]
 
 jobs:
-  bst1-79-0:
+  bst1-80-0:
     runs-on: ubuntu-latest
     env:
       METALL_LIMIT_MAKE_PARALLELS: 8
@@ -12,9 +12,9 @@ jobs:
       - name: Test
         run: |
           pushd /dev/shm
-          wget -q https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz
+          wget -q https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz
           mkdir boost
-          tar xf boost_1_79_0.tar.gz -C boost --strip-components 1
+          tar xf boost_1_80_0.tar.gz -C boost --strip-components 1
           export BOOST_ROOT=${PWD}/boost
           popd
           export METALL_TEST_DIR=${GITHUB_JOB}

.gitlab-ci.yml

@@ -27,6 +27,7 @@ install_boost:
   script:
     - hostname
     - pwd
+#    - spack install [email protected]
 #    - spack install [email protected]
 #    - spack install [email protected]
 #    - spack install [email protected]
@@ -60,6 +61,12 @@ install_boost:
     - export METALL_TEST_DIR="/dev/shm/metall_test-${CI_CONCURRENT_ID}-${CI_PIPELINE_IID}"
     - srun -N1 -ppdebug bash ./scripts/CI/build_and_test.sh
 
+build_gcc10.2.1_bst1.80.0:
+  extends: .build
+  variables:
+    GCC_VERSION: "10.2.1"
+    BOOST_VERSION: "1.80.0"
+
 build_gcc10.2.1_bst1.79.0:
   extends: .build
   variables:

CMakeLists.txt

@@ -17,7 +17,7 @@ endif()
 # Metall general configuration
 # -------------------------------------------------------------------------------- #
 project(Metall
-        VERSION 0.22
+        VERSION 0.23
         DESCRIPTION "A persistent memory allocator for data-centric analytics"
         HOMEPAGE_URL "https://github.com/LLNL/metall")
 

bench/CMakeLists.txt

@@ -2,4 +2,6 @@ add_subdirectory(simple_alloc)
 add_subdirectory(adjacency_list)
 add_subdirectory(bfs)
 add_subdirectory(rand_engine)
-add_subdirectory(mapping)
+add_subdirectory(mapping)
+add_subdirectory(container)
+add_subdirectory(offset_ptr)

bench/container/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_metall_executable(run_vector_bench run_vector_bench.cpp)
+add_metall_executable(run_map_bench run_map_bench.cpp)
+add_metall_executable(run_unordered_map_bench run_unordered_map_bench.cpp)

bench/container/bench_common.hpp

@@ -0,0 +1,63 @@
+// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers.
+// See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+#ifndef METALL_BENCH_CONTAINER_BENCH_COMMON_HPP
+#define METALL_BENCH_CONTAINER_BENCH_COMMON_HPP
+
+#include <vector>
+#include <utility>
+#include <random>
+
+#include <metall/utility/random.hpp>
+
+#include "../bench/adjacency_list/edge_generator/rmat_edge_generator.hpp"
+
+namespace mdtl = metall::mtlldetail;
+
+using rmat_generator_type = edge_generator::rmat_edge_generator;
+
+void gen_edges(const std::size_t vertex_scale,
+               const std::size_t num_edges,
+               std::vector<std::pair<uint64_t, uint64_t>> &buf) {
+  rmat_generator_type rmat_generator(123, vertex_scale, num_edges, 0.57, 0.19, 0.19, true, false);
+
+  buf.reserve(num_edges);
+  for (auto itr = rmat_generator.begin(); itr != rmat_generator.end(); ++itr) {
+    buf.emplace_back(*itr);
+  }
+}
+
+void gen_random_values(const std::size_t num_values,
+                       std::vector<std::pair<uint64_t, uint64_t>> &buf) {
+  metall::utility::rand_1024 rnd_generator(std::random_device{}());
+
+  buf.reserve(num_values);
+  for (std::size_t i = 0; i < num_values; ++i) {
+    buf.emplace_back(rnd_generator(), rnd_generator());
+  }
+}
+
+template <typename input_container_type, typename inserter_func>
+void run_bench(const input_container_type &inputs, const inserter_func &inserter) {
+  const auto start = mdtl::elapsed_time_sec();
+  for (const auto &kv: inputs) {
+    inserter(kv);
+  }
+  const auto elapsed_time = mdtl::elapsed_time_sec(start);
+  std::cout << "Took (s)\t" << elapsed_time << std::endl;
+}
+
+template <typename input_container_type, typename preprocessor_func, typename inserter_func>
+void run_bench(const input_container_type &inputs, const preprocessor_func& preprocessor, const inserter_func &inserter) {
+  const auto start = mdtl::elapsed_time_sec();
+  preprocessor();
+  for (const auto &kv: inputs) {
+    inserter(kv);
+  }
+  const auto elapsed_time = mdtl::elapsed_time_sec(start);
+  std::cout << "Took (s)\t" << elapsed_time << std::endl;
+}
+
+#endif //METALL_BENCH_CONTAINER_BENCH_COMMON_HPP

bench/container/run_map_bench.cpp

@@ -0,0 +1,67 @@
+// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers.
+// See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+/// \brief Benchmarks the STL map container using different allocators.
+/// Usage:
+/// ./run_map_bench
+/// # modify the values in the main(), if needed.
+
+#include <iostream>
+#include <map>
+#include <boost/container/map.hpp>
+#include <metall/container/map.hpp>
+#include <metall/metall.hpp>
+#include <metall/detail/time.hpp>
+
+#include "bench_common.hpp"
+
+int main() {
+  std::size_t scale = 17;
+  std::size_t num_inputs = (1ULL << scale) * 16;
+  std::vector<std::pair<uint64_t, uint64_t>> inputs;
+
+  //gen_edges(scale, num_inputs, inputs);
+  gen_random_values(num_inputs, inputs);
+  std::cout << "Generated inputs\t" << inputs.size() << std::endl;
+
+  {
+    std::map<uint64_t, uint64_t> map;
+
+    const auto start = mdtl::elapsed_time_sec();
+    for (const auto &kv: inputs) {
+      map[kv.first];
+      map[kv.second];
+    }
+    const auto elapsed_time = mdtl::elapsed_time_sec(start);
+    std::cout << "map took (s)\t" << elapsed_time << std::endl;
+  }
+
+  {
+    boost::container::map<uint64_t, uint64_t> map;
+
+    const auto start = mdtl::elapsed_time_sec();
+    for (const auto &kv: inputs) {
+      map[kv.first];
+      map[kv.second];
+    }
+    const auto elapsed_time = mdtl::elapsed_time_sec(start);
+    std::cout << "Boost map took (s)\t" << elapsed_time << std::endl;
+  }
+
+  {
+    metall::manager mngr(metall::create_only, "/tmp/metall");
+    metall::container::map<uint64_t, uint64_t> map(mngr.get_allocator());
+
+    const auto start = mdtl::elapsed_time_sec();
+    for (const auto &kv: inputs) {
+      map[kv.first];
+      map[kv.second];
+    }
+    const auto elapsed_time = mdtl::elapsed_time_sec(start);
+    std::cout << "Boost map with Metall took (s)\t" << elapsed_time << std::endl;
+  }
+
+  return 0;
+}

bench/container/run_unordered_map_bench.cpp

@@ -0,0 +1,67 @@
+// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers.
+// See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+/// \brief Benchmarks the STL map container using different allocators.
+/// Usage:
+/// ./run_map_bench
+/// # modify the values in the main(), if needed.
+
+#include <iostream>
+#include <unordered_map>
+#include <boost/unordered_map.hpp>
+#include <metall/container/unordered_map.hpp>
+#include <metall/metall.hpp>
+#include <metall/detail/time.hpp>
+
+#include "bench_common.hpp"
+
+int main() {
+  std::size_t scale = 17;
+  std::size_t num_inputs = (1ULL << scale) * 16;
+  std::vector<std::pair<uint64_t, uint64_t>> inputs;
+
+  //gen_edges(scale, num_inputs, inputs);
+  gen_random_values(num_inputs, inputs);
+  std::cout << "Generated inputs\t" << inputs.size() << std::endl;
+
+  {
+    std::unordered_map<uint64_t, uint64_t> map;
+
+    const auto start = mdtl::elapsed_time_sec();
+    for (const auto &kv: inputs) {
+      map[kv.first];
+      map[kv.second];
+    }
+    const auto elapsed_time = mdtl::elapsed_time_sec(start);
+    std::cout << "unordered_map took (s)\t" << elapsed_time << std::endl;
+  }
+
+  {
+    boost::unordered_map<uint64_t, uint64_t> map;
+
+    const auto start = mdtl::elapsed_time_sec();
+    for (const auto &kv: inputs) {
+      map[kv.first];
+      map[kv.second];
+    }
+    const auto elapsed_time = mdtl::elapsed_time_sec(start);
+    std::cout << "Boost unordered_map took (s)\t" << elapsed_time << std::endl;
+  }
+
+  {
+    metall::manager mngr(metall::create_only, "/tmp/metall");
+    metall::container::unordered_map<uint64_t, uint64_t> map(mngr.get_allocator());
+
+    const auto start = mdtl::elapsed_time_sec();
+    for (const auto &kv: inputs) {
+      map[kv.first];
+      map[kv.second];
+    }
+    const auto elapsed_time = mdtl::elapsed_time_sec(start);
+    std::cout << "Boost unordered_map with Metall took (s)\t" << elapsed_time << std::endl;
+  }
+
+  return 0;
+}

bench/container/run_vector_bench.cpp

@@ -0,0 +1,78 @@
+// Copyright 2022 Lawrence Livermore National Security, LLC and other Metall Project Developers.
+// See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+/// \brief Benchmarks the STL map container using different allocators.
+/// Usage:
+/// ./run_map_bench
+/// # modify the values in the main(), if needed.
+
+#include <iostream>
+#include <vector>
+#include <boost/container/vector.hpp>
+#include <metall/container/vector.hpp>
+#include <metall/metall.hpp>
+#include <metall/detail/time.hpp>
+
+#include "bench_common.hpp"
+
+int main() {
+  std::size_t scale = 22;
+  std::size_t num_inputs = (1ULL << scale) * 16;
+  std::vector<std::pair<uint64_t, uint64_t>> inputs;
+
+  gen_random_values(num_inputs, inputs);
+  std::cout << "Generated inputs\t" << inputs.size() << std::endl;
+
+  {
+    std::cout << "vector (push_back)" << std::endl;
+    std::vector<std::pair<uint64_t, uint64_t>> vec;
+    run_bench(inputs, [&vec](const auto &kv) { vec.push_back(kv); });
+  }
+
+  {
+    std::cout << "Boost vector (push_back)" << std::endl;
+    boost::container::vector<std::pair < uint64_t, uint64_t>>
+    vec;
+    run_bench(inputs, [&vec](const auto &kv) { vec.push_back(kv); });
+  }
+
+  {
+    std::cout << "Boost vector (push_back) with Metall" << std::endl;
+    metall::manager mngr(metall::create_only, "/tmp/metall");
+    metall::container::vector<std::pair<uint64_t, uint64_t>> vec(mngr.get_allocator());
+    run_bench(inputs, [&vec](const auto &kv) { vec.push_back(kv); });
+  }
+  std::cout << std::endl;
+
+  {
+    std::cout << "vector ([])" << std::endl;
+    std::vector<std::pair<uint64_t, uint64_t>> vec;
+    std::size_t index = 0;
+    run_bench(inputs,
+              [&vec, &inputs]() { vec.resize(inputs.size()); },
+              [&vec, &index](const auto &kv) { vec[index++] = kv; });
+  }
+
+  {
+    std::cout << "Boost ([]) vector" << std::endl;
+    boost::container::vector<std::pair < uint64_t, uint64_t>>
+    vec;
+    std::size_t index = 0;
+    run_bench(inputs,
+              [&vec, &inputs]() { vec.resize(inputs.size()); },
+              [&vec, &index](const auto &kv) { vec[index++] = kv; });
+  }
+
+  {
+    std::cout << "Boost vector ([]) with Metall" << std::endl;
+    metall::manager mngr(metall::create_only, "/tmp/metall");
+    metall::container::vector<std::pair<uint64_t, uint64_t>> vec(mngr.get_allocator());
+    std::size_t index = 0;
+    run_bench(inputs,
+              [&vec, &inputs]() { vec.resize(inputs.size()); },
+              [&vec, &index](const auto &kv) { vec[index++] = kv; });
+  }
+  return 0;
+}

bench/mapping/run_mapping_bench.cpp

@@ -1,3 +1,13 @@
+// Copyright 2019 Lawrence Livermore National Security, LLC and other Metall Project Developers.
+// See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+/// \brief Benchmarks file mapping backends.
+/// Usage:
+/// ./run_mapping_bench
+/// # modify some values in the main function, if needed.
+
 #include <iostream>
 #include <random>
 #include <string>

bench/offset_ptr/CMakeLists.txt

@@ -0,0 +1 @@
+add_metall_executable(run_offset_ptr_bench run_offset_ptr_bench.cpp)