From f5bb7137645d9068609fb0544749ddcad83e973e Mon Sep 17 00:00:00 2001 From: Jerry Watkins Date: Fri, 13 Dec 2024 16:16:59 -0700 Subject: [PATCH] Fix nightly GPU failures after kokkos 4.5.0 - remove create_copy() and create temp workspace - add Helmholtz back in --- src/PHAL_Utilities_Def.hpp | 31 ------------------- src/corePDEs/evaluators/PHAL_HeatEqResid.hpp | 1 + .../evaluators/PHAL_HeatEqResid_Def.hpp | 9 +++--- .../evaluators/PHAL_HelmholtzResid.hpp | 3 ++ .../evaluators/PHAL_HelmholtzResid_Def.hpp | 13 ++++++-- src/demoPDEs/evaluators/PHAL_PoissonResid.hpp | 2 ++ .../evaluators/PHAL_PoissonResid_Def.hpp | 8 +++-- tests/demoPDEs/CMakeLists.txt | 1 + 8 files changed, 27 insertions(+), 41 deletions(-) diff --git a/src/PHAL_Utilities_Def.hpp b/src/PHAL_Utilities_Def.hpp index be65745a8a..453f060671 100644 --- a/src/PHAL_Utilities_Def.hpp +++ b/src/PHAL_Utilities_Def.hpp @@ -260,35 +260,4 @@ void scale (ArrayT& a, const T& val) { loop(sl, a); } -template< class T , class ... P > -inline -typename std::enable_if< - !Kokkos::is_dynrankview_fad>::value, - typename Kokkos::DynRankView::non_const_type >::type -create_copy( const std::string& name, - const Kokkos::DynRankView & src ) -{ - using dst_type = typename Kokkos::DynRankView::non_const_type; - auto layout = Kokkos::Impl::reconstructLayout(src.layout(), src.rank()); - return dst_type( name , layout ); -} - -template< class T , class ... P > -inline -typename std::enable_if< - Kokkos::is_dynrankview_fad>::value, - typename Kokkos::DynRankView::non_const_type >::type -create_copy( const std::string& /* name */, - const Kokkos::DynRankView & src ) -{ - using Dst = typename Kokkos::DynRankView::non_const_type; - auto sm = src.impl_map(); - auto sl = sm.layout(); - auto fad_rank = src.rank(); - sl.dimension[fad_rank] = sm.dimension_scalar(); - auto real_rank = fad_rank + 1; - auto ml = Kokkos::Impl::reconstructLayout(sl, real_rank); - return Dst ( src.label(), ml ); -} - } // namespace PHAL diff --git a/src/corePDEs/evaluators/PHAL_HeatEqResid.hpp b/src/corePDEs/evaluators/PHAL_HeatEqResid.hpp index ccf626a00f..7424c4bada 100644 --- a/src/corePDEs/evaluators/PHAL_HeatEqResid.hpp +++ b/src/corePDEs/evaluators/PHAL_HeatEqResid.hpp @@ -60,6 +60,7 @@ class HeatEqResid : public PHX::EvaluatorWithBaseImpl, bool haverhoCp; unsigned int numQPs, numDims, numNodes, worksetSize; Kokkos::DynRankView flux; + Kokkos::DynRankView neg_source; Kokkos::DynRankView aterm; Kokkos::DynRankView convection; }; diff --git a/src/corePDEs/evaluators/PHAL_HeatEqResid_Def.hpp b/src/corePDEs/evaluators/PHAL_HeatEqResid_Def.hpp index 00d9d1e3f1..a6c20a7c0b 100644 --- a/src/corePDEs/evaluators/PHAL_HeatEqResid_Def.hpp +++ b/src/corePDEs/evaluators/PHAL_HeatEqResid_Def.hpp @@ -107,9 +107,10 @@ postRegistrationSetup(typename Traits::SetupData /* d */, this->utils.setFieldData(TResidual,fm); // Allocate workspace - flux = Kokkos::createDynRankView(Temperature.get_view(), "XXX", worksetSize, numQPs, numDims); - if (haveAbsorption) aterm = Kokkos::createDynRankView(Temperature.get_view(), "XXX", worksetSize, numQPs); - if (haveConvection) convection = Kokkos::createDynRankView(Temperature.get_view(), "XXX", worksetSize, numQPs); + flux = Kokkos::createDynRankView(Temperature.get_view(), "flux", worksetSize, numQPs, numDims); + if (haveSource) neg_source = Kokkos::createDynRankView(Temperature.get_view(), "neg_source", worksetSize, numQPs); + if (haveAbsorption) aterm = Kokkos::createDynRankView(Temperature.get_view(), "aterm", worksetSize, numQPs); + if (haveConvection) convection = Kokkos::createDynRankView(Temperature.get_view(), "convection", worksetSize, numQPs); } //********************************************************************** @@ -124,8 +125,6 @@ evaluateFields(typename Traits::EvalData workset) FST::integrate(TResidual.get_view(), flux, wGradBF.get_view(), false); // "false" overwrites if (haveSource) { - auto neg_source = PHAL::create_copy("neg_source", Source.get_view()); - for (size_t i =0; i< Source.extent(0); i++) for (size_t j =0; j< Source.extent(1); j++) neg_source(i,j) = Source(i,j) * -1.0; diff --git a/src/demoPDEs/evaluators/PHAL_HelmholtzResid.hpp b/src/demoPDEs/evaluators/PHAL_HelmholtzResid.hpp index aef6ff2a3e..06b491174d 100644 --- a/src/demoPDEs/evaluators/PHAL_HelmholtzResid.hpp +++ b/src/demoPDEs/evaluators/PHAL_HelmholtzResid.hpp @@ -54,7 +54,10 @@ class HelmholtzResid : public PHX::EvaluatorWithBaseImpl, bool haveSource; + unsigned int worksetSize, numQPs; ScalarT ksqr; + Kokkos::DynRankView U_ksqr; + Kokkos::DynRankView V_ksqr; // Output: PHX::MDField UResidual; diff --git a/src/demoPDEs/evaluators/PHAL_HelmholtzResid_Def.hpp b/src/demoPDEs/evaluators/PHAL_HelmholtzResid_Def.hpp index 28e82730aa..29d7b12ca1 100644 --- a/src/demoPDEs/evaluators/PHAL_HelmholtzResid_Def.hpp +++ b/src/demoPDEs/evaluators/PHAL_HelmholtzResid_Def.hpp @@ -61,6 +61,12 @@ HelmholtzResid(const Teuchos::ParameterList& p) : p.get< Teuchos::RCP >("Parameter Library"); this->registerSacadoParameter("Ksqr", paramLib); + Teuchos::RCP qp_scalar_dl = + p.get< Teuchos::RCP >("QP Scalar Data Layout"); + std::vector dims; + qp_scalar_dl->dimensions(dims); + worksetSize = dims[0]; + numQPs = dims[1]; } //********************************************************************** @@ -81,6 +87,10 @@ postRegistrationSetup(typename Traits::SetupData /* d */, } this->utils.setFieldData(UResidual,fm); this->utils.setFieldData(VResidual,fm); + + // Allocate workspace + U_ksqr = Kokkos::createDynRankView(U.get_view(), "U_ksqr", worksetSize, numQPs); + V_ksqr = Kokkos::createDynRankView(U.get_view(), "V_ksqr", worksetSize, numQPs); } //********************************************************************** @@ -102,9 +112,6 @@ evaluateFields(typename Traits::EvalData /* workset */) FST::integrate(VResidual.get_view(), VSource.get_view(), wBF.get_view(), true); } - auto U_ksqr = create_copy("U_ksqr", U.get_view()); - auto V_ksqr = create_copy("V_ksqr", V.get_view()); - RST::scale(U_ksqr, U.get_view(), ksqr); RST::scale(V_ksqr, V.get_view(), ksqr); diff --git a/src/demoPDEs/evaluators/PHAL_PoissonResid.hpp b/src/demoPDEs/evaluators/PHAL_PoissonResid.hpp index 6ba2c9994c..a405d122b4 100644 --- a/src/demoPDEs/evaluators/PHAL_PoissonResid.hpp +++ b/src/demoPDEs/evaluators/PHAL_PoissonResid.hpp @@ -44,6 +44,8 @@ class PoissonResid : public PHX::EvaluatorWithBaseImpl, PHX::MDField Source; bool haveSource; + unsigned int worksetSize, numQPs; + Kokkos::DynRankView neg_source; // Output: PHX::MDField PhiResidual; diff --git a/src/demoPDEs/evaluators/PHAL_PoissonResid_Def.hpp b/src/demoPDEs/evaluators/PHAL_PoissonResid_Def.hpp index f33ffb4bc5..4f3c506b3e 100644 --- a/src/demoPDEs/evaluators/PHAL_PoissonResid_Def.hpp +++ b/src/demoPDEs/evaluators/PHAL_PoissonResid_Def.hpp @@ -23,7 +23,9 @@ PoissonResid(const Teuchos::ParameterList& p, Source (p.get ("Source Name"), dl->qp_scalar ), haveSource (p.get("Have Source")), PhiResidual (p.get ("Residual Name"), dl->node_scalar ), - PhiFlux (p.get ("Flux QP Variable Name"), dl->qp_gradient) + PhiFlux (p.get ("Flux QP Variable Name"), dl->qp_gradient), + worksetSize (dl->qp_scalar->extent(0)), + numQPs (dl->qp_scalar->extent(1)) { this->addDependentField(wBF); //this->addDependentField(Potential); @@ -53,6 +55,9 @@ postRegistrationSetup(typename Traits::SetupData /* d */, if (haveSource) this->utils.setFieldData(Source,fm); this->utils.setFieldData(PhiResidual,fm); + + // Allocate workspace + if (haveSource) neg_source = Kokkos::createDynRankView(Source.get_view(), "neg_source", worksetSize, numQPs); } //********************************************************************** @@ -67,7 +72,6 @@ evaluateFields(typename Traits::EvalData /* workset */) FST::integrate(PhiResidual.get_view(), PhiFlux.get_view(), wGradBF.get_view(), false); // "false" overwrites - auto neg_source = PHAL::create_copy("neg_Source", Source.get_view()); if (haveSource) { for (unsigned int i=0; i