Added minimal example

.gitignore

@@ -1,3 +1,4 @@
 build
 external_dependencies/build
 external_dependencies/install
+.spack-env

CMakeLists.txt

@@ -54,3 +54,9 @@ add_executable(
   ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator_executor_stream_test.cpp)
 target_link_libraries(allocator_executor_stream_test
   PRIVATE HPX::hpx Kokkos::kokkos HPXKokkos::hpx_kokkos buffer_manager)
+
+add_executable(
+  minimal_parallel_for
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/minimal_parallel_for.cpp)
+target_link_libraries(minimal_parallel_for
+  PRIVATE HPX::hpx Kokkos::kokkos HPXKokkos::hpx_kokkos buffer_manager)

include/buffer_manager.hpp

@@ -159,6 +159,7 @@ class buffer_recycler {
           assert(std::get<1>(tuple) == number_of_elements);
           assert(std::get<2>(tuple) >= 1);
           std::get<2>(tuple)--; // decrease usage counter
+          std::cout << "Usage counter is now " << std::get<2>(tuple) << std::endl;
           if (std::get<2>(tuple) == 0) { // not used anymore?
             // move to the unused_buffer list 
             instance->unused_buffer_list.push_front(tuple);
@@ -174,6 +175,7 @@ class buffer_recycler {
           assert(std::get<1>(tuple) == number_of_elements);
           assert(std::get<2>(tuple) >= 1);
           std::get<2>(tuple)++; // increase usage counter
+          std::cout << "Usage counter is now " << std::get<2>(tuple) << std::endl;
         }
 
       private:

include/kokkos_buffer_util.hpp

@@ -16,20 +16,24 @@ class recycled_view : public kokkos_type
                                   total_elements(kokkos_type::required_allocation_size(args...) / sizeof(element_type))
     {
         //std::cout << "Got buffer for " << total_elements << std::endl;
+        std::cout << "Creating view" << std::endl;
     }
 
     recycled_view(const recycled_view<kokkos_type, alloc_type, element_type> &other) : kokkos_type(other)
     {
         total_elements = other.total_elements;
         // std::cerr << "copy" << std::endl;
         allocator.increase_usage_counter(other.data(), other.total_elements);
+        std::cout << "Copying view" << std::endl;
+
     }
 
     recycled_view<kokkos_type, alloc_type, element_type> &operator=(const recycled_view<kokkos_type, alloc_type, element_type> &other)
     {
         kokkos_type::operator=(other);
         total_elements = other.total_elements;
         allocator.increase_usage_counter(other.data(), other.total_elements);
+        std::cout << "Copying view" << std::endl;
         return *this;
     }
 
@@ -38,6 +42,7 @@ class recycled_view : public kokkos_type
         total_elements = other.total_elements;
         // so that is doesn't matter if deallocate is called in the moved-from object
         allocator.increase_usage_counter(other.data(), other.total_elements);
+        std::cout << "Moving view" << std::endl;
     }
 
     recycled_view<kokkos_type, alloc_type, element_type> &operator=(recycled_view<kokkos_type, alloc_type, element_type> &&other)
@@ -46,11 +51,13 @@ class recycled_view : public kokkos_type
         total_elements = other.total_elements;
         // so that is doesn't matter if deallocate is called in the moved-from object
         allocator.increase_usage_counter(other.data(), other.total_elements);
+        std::cout << "Moving view" << std::endl;
         return *this;
     }
 
-    ~recycled_view(void)
+    virtual ~recycled_view(void)
     {
+        std::cout << "Dellocating view" << std::endl;
         allocator.deallocate(this->data(), total_elements);
     }
 

scripts/build_kokkos.sh

@@ -20,6 +20,6 @@ INSTALL_DIR=${SCRIPTS_DIR}/../external_dependencies/install/kokkos-${CMAKE_BUILD
 
 mkdir -p ${BUILD_DIR}
 pushd ${BUILD_DIR}
-cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DHPX_DIR=${HPX_ROOT} -DKokkos_CXX_STANDARD=14 -DKokkos_ARCH_PASCAL61=ON ${CURRENT_CUDA_ARCH_FLAG} -DKokkos_ARCH_HSW=ON -DKokkos_ENABLE_HPX=ON -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_SERIAL=OFF -DKokkos_ENABLE_TESTS=OFF -DKokkos_ENABLE_HPX_ASYNC_DISPATCH=ON -DKokkos_ENABLE_INTERNAL_FENCES=OFF -DKokkos_ENABLE_CUDA_LAMBDA=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} ${SOURCE_DIR}
+cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DHPX_DIR=${HPX_ROOT} -DKokkos_CXX_STANDARD=14 -DKokkos_ARCH_PASCAL61=ON ${CURRENT_CUDA_ARCH_FLAG} -DKokkos_ARCH_HSW=ON -DKokkos_ENABLE_HPX=ON -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_TESTS=OFF -DKokkos_ENABLE_HPX_ASYNC_DISPATCH=ON -DKokkos_ENABLE_INTERNAL_FENCES=OFF -DKokkos_ENABLE_CUDA_LAMBDA=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} ${SOURCE_DIR}
 make -j$(nproc) install
 popd

src/minimal_parallel_for.cpp

@@ -0,0 +1,51 @@
+
+#include <hpx/hpx_main.hpp>
+#include <hpx/include/async.hpp>
+#include <hpx/include/lcos.hpp>
+
+#include <hpx/kokkos.hpp>
+
+#include <Kokkos_Core.hpp>
+#include <cstdio>
+#include <typeinfo>
+
+#include "../include/buffer_manager.hpp"
+#include "../include/cuda_buffer_util.hpp"
+#include "../include/kokkos_buffer_util.hpp"
+#include <hpx/timing/high_resolution_timer.hpp>
+#include <memory>
+
+constexpr size_t view_size_0 = 10;
+constexpr size_t view_size_1 = 50;
+template <class T>
+using kokkos_um_array = Kokkos::View<T**, Kokkos::HostSpace, Kokkos::MemoryUnmanaged>;
+template <class T>
+using recycled_host_view = recycled_view<kokkos_um_array<T>, recycle_std<T>, T>;
+
+
+template <typename Executor, typename ViewType>
+auto get_iteration_policy(const Executor&& executor, const ViewType& view_to_iterate){ 
+    return get_iteration_policy(executor, view_to_iterate);
+}
+
+int main(int argc, char *argv[])
+{
+    constexpr size_t passes = 1;
+    for (size_t pass = 0; pass < passes; pass++)
+    {
+      recycled_host_view<double> hostView(view_size_0,view_size_1);
+
+      // works - usage counter goes up to 9 for the hostView
+      // auto host_space = hpx::kokkos::make_execution_space<Kokkos::Serial>(); 
+
+      // broken - usage counter goes up to 13 for the hostView - goes only down to 2
+      auto host_space = hpx::kokkos::make_execution_space<Kokkos::DefaultHostExecutionSpace>();
+      auto policy_host = get_iteration_policy(host_space, hostView);
+      Kokkos::parallel_for(
+          "host init",
+          policy_host,
+          KOKKOS_LAMBDA(int n, int o) {
+            hostView(n, o) = 1.0;
+          });
+    }
+}