From ff0ebced6d4fb9ef5db122184e4d85b3547fc6ee Mon Sep 17 00:00:00 2001 From: Daniel Holladay Date: Fri, 31 Mar 2023 13:28:16 -0600 Subject: [PATCH 1/5] Initial commit to remove kokkos specific code for copying views, based on new ports-of-call capabilities. --- spiner/databox.hpp | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/spiner/databox.hpp b/spiner/databox.hpp index feb83880a..35aece6e3 100644 --- a/spiner/databox.hpp +++ b/spiner/databox.hpp @@ -296,35 +296,24 @@ class DataBox { // TODO(JMM): Add more code for more portability strategies DataBox getOnDevice() const { // getOnDevice is always a deep copy -#ifdef PORTABILITY_STRATEGY_KOKKOS - using HS = Kokkos::HostSpace; - using DMS = Kokkos::DefaultExecutionSpace::memory_space; - constexpr const bool execution_is_host{ - Kokkos::SpaceAccessibility::accessible}; - if (execution_is_host) { - DataBox a; - a.copy(*this); // a.copy handles setting allocation status - return a; - } else { - using memUnmanaged = Kokkos::MemoryUnmanaged; - using HostView_t = Kokkos::View; - using DeviceView_t = Kokkos::View; - using Kokkos::deep_copy; - T *device_data = (T *)PORTABLE_MALLOC(sizeBytes()); - DeviceView_t devView(device_data, dataView_.GetSize()); - HostView_t hostView(data_, dataView_.GetSize()); - deep_copy(devView, hostView); - DataBox a{devView.data(), dim(6), dim(5), dim(4), - dim(3), dim(2), dim(1)}; - a.copyShape(*this); +#ifdef __CUDACC__ + constexpr const bool execution_is_host = false; +#else + constexpr const bool execution_is_host = true; +#endif // __CUDACC__ + // create device memory (host memory if no device) + T *device_data = (T *)PORTABLE_MALLOC(sizeBytes()); + // copy to device + portableCopyToDevice(device_data, data_, sizeBytes()); + // create new databox of size size + DataBox a{device_data, dim(6), dim(5), dim(4), + dim(3), dim(2), dim(1)}; + a.copyShape(*this); + // this may need logic for if host? JMM thoughts? + if (!execution_is_host) { a.status_ = DataStatus::AllocatedDevice; - return a; } -#else // no kokkos - DataBox a; - a.copy(*this); // a.copy handles allocation status. return a; -#endif // kokkos } // TODO(JMM): Potentially use this for device-free From b6f28996fab427869a5f59a7103e0517e252699b Mon Sep 17 00:00:00 2001 From: Daniel Holladay Date: Mon, 3 Apr 2023 14:26:00 -0600 Subject: [PATCH 2/5] Using new POC variable for host/device execution query. --- spiner/databox.hpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/spiner/databox.hpp b/spiner/databox.hpp index 35aece6e3..7c92203ec 100644 --- a/spiner/databox.hpp +++ b/spiner/databox.hpp @@ -296,21 +296,17 @@ class DataBox { // TODO(JMM): Add more code for more portability strategies DataBox getOnDevice() const { // getOnDevice is always a deep copy -#ifdef __CUDACC__ - constexpr const bool execution_is_host = false; -#else - constexpr const bool execution_is_host = true; -#endif // __CUDACC__ // create device memory (host memory if no device) T *device_data = (T *)PORTABLE_MALLOC(sizeBytes()); // copy to device portableCopyToDevice(device_data, data_, sizeBytes()); // create new databox of size size - DataBox a{device_data, dim(6), dim(5), dim(4), - dim(3), dim(2), dim(1)}; + DataBox a{device_data, dim(6), dim(5), dim(4), dim(3), dim(2), dim(1)}; a.copyShape(*this); // this may need logic for if host? JMM thoughts? - if (!execution_is_host) { + if (PortsOfCall::EXECUTION_IS_HOST) { + a.status_ = DataStatus::AllocatedHost; + } else { a.status_ = DataStatus::AllocatedDevice; } return a; From 57d99daf63bb3710e093afd5b7038182cb2ba91d Mon Sep 17 00:00:00 2001 From: Daniel Holladay Date: Tue, 4 Apr 2023 17:49:55 -0600 Subject: [PATCH 3/5] Udpate ports-of-call version pulled in cmake. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b33add3a4..307e4085d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,7 +87,8 @@ spiner_content_declare(ports-of-call NAMESPACE spinerDeps GIT_REPO https://github.com/lanl/ports-of-call # NOTE: main as of Nov. 28 2022 - GIT_TAG 3fffe08408b1ee754d0567a3de63cb47ce97d9f7 + #GIT_TAG 3fffe08408b1ee754d0567a3de63cb47ce97d9f7 + GIT_TAG v1.5.1 ) if(SPINER_USE_HDF) From ff939806d671f353a2aeca74106ec7b65fb0773b Mon Sep 17 00:00:00 2001 From: Daniel Holladay Date: Wed, 5 Apr 2023 07:29:28 -0600 Subject: [PATCH 4/5] Updating spackages. --- spack-repo/packages/ports-of-call/package.py | 1 + spack-repo/packages/spiner/package.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/spack-repo/packages/ports-of-call/package.py b/spack-repo/packages/ports-of-call/package.py index 73f737b44..fab9881a5 100644 --- a/spack-repo/packages/ports-of-call/package.py +++ b/spack-repo/packages/ports-of-call/package.py @@ -16,6 +16,7 @@ class PortsOfCall(CMakePackage): maintainers = ["rbberger"] version("main", branch="main") + version("1.5.1", sha256="b1f0232cd6d2aac65385d77cc061ec5035283ea50d0f167e7003eae034effb78") version("1.4.1", sha256="82d2c75fcca8bd613273fd4126749df68ccc22fbe4134ba673b4275f9972b78d") version("1.4.0", sha256="e08ae556b7c30d14d77147d248d118cf5343a2e8c0847943385c602394bda0fa") version("1.3.0", sha256="54b4a62539c23b1a345dd87c1eac65f4f69db4e50336cd81a15a627ce80ce7d9") diff --git a/spack-repo/packages/spiner/package.py b/spack-repo/packages/spiner/package.py index bd2901ab9..eacfd9d93 100644 --- a/spack-repo/packages/spiner/package.py +++ b/spack-repo/packages/spiner/package.py @@ -53,7 +53,7 @@ class Spiner(CMakePackage, CudaPackage): depends_on("cmake@3.23:", when="@1.6.0:") depends_on("catch2@2.13.4:2.13.9") depends_on("ports-of-call@1.2.0:", when="@:1.5.1") - depends_on("ports-of-call@1.3.0:", when="@1.6.0:") + depends_on("ports-of-call@1.5.1:", when="@1.6.0:") # Currently the raw cuda backend of ports-of-call is not supported. depends_on("ports-of-call portability_strategy=Kokkos", when="@:1.5.1 +kokkos") @@ -63,7 +63,7 @@ class Spiner(CMakePackage, CudaPackage): for _flag in ("~cuda", "+cuda", "~openmp", "+openmp"): depends_on("kokkos@3.3.00: " + _flag, when="+kokkos" + _flag) depends_on( - "kokkos@3.3.00: ~shared+wrapper+cuda_lambda+cuda_constexpr+cuda_relocatable_device_code", + "kokkos@3.3.00: ~shared+wrapper+cuda_lambda+cuda_constexpr", when="+cuda+kokkos", ) From f470751f04a7ddfc61813be2edf55614458390f0 Mon Sep 17 00:00:00 2001 From: Daniel Holladay Date: Wed, 5 Apr 2023 10:18:21 -0600 Subject: [PATCH 5/5] Comment cleanup. --- CMakeLists.txt | 3 +-- spiner/databox.hpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 307e4085d..fc8c1c3f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,8 +86,7 @@ include(content) spiner_content_declare(ports-of-call NAMESPACE spinerDeps GIT_REPO https://github.com/lanl/ports-of-call - # NOTE: main as of Nov. 28 2022 - #GIT_TAG 3fffe08408b1ee754d0567a3de63cb47ce97d9f7 + # most recent relase as of April 05, 2023 GIT_TAG v1.5.1 ) diff --git a/spiner/databox.hpp b/spiner/databox.hpp index 7c92203ec..20e565a96 100644 --- a/spiner/databox.hpp +++ b/spiner/databox.hpp @@ -294,7 +294,6 @@ class DataBox { return indices_[i]; } - // TODO(JMM): Add more code for more portability strategies DataBox getOnDevice() const { // getOnDevice is always a deep copy // create device memory (host memory if no device) T *device_data = (T *)PORTABLE_MALLOC(sizeBytes()); @@ -303,7 +302,7 @@ class DataBox { // create new databox of size size DataBox a{device_data, dim(6), dim(5), dim(4), dim(3), dim(2), dim(1)}; a.copyShape(*this); - // this may need logic for if host? JMM thoughts? + // set correct allocation status of the new databox if (PortsOfCall::EXECUTION_IS_HOST) { a.status_ = DataStatus::AllocatedHost; } else {