diff --git a/libs/runcleo/creategbxs.hpp b/libs/runcleo/creategbxs.hpp index f0f2276e..ee263757 100644 --- a/libs/runcleo/creategbxs.hpp +++ b/libs/runcleo/creategbxs.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -240,6 +241,9 @@ void print_gbxs(const viewh_constgbx gbxs); * This function creates Gridboxes based on the provided gridbox maps and initial conditions, * and given super-droplets. * + * Kokkos::Profiling are null pointers unless a Kokkos profiler library has been + * exported to "KOKKOS_TOOLS_LIBS" prior to runtime so the lib gets dynamically loaded. + * * @tparam GbxMaps Type representing Gridbox Maps. * @tparam GbxInitConds Type representing Gridbox initial conditions. * @@ -252,6 +256,8 @@ void print_gbxs(const viewh_constgbx gbxs); template dualview_gbx create_gbxs(const GbxMaps &gbxmaps, const GbxInitConds &gbxic, const viewd_supers totsupers) { + Kokkos::Profiling::ScopedRegion region("init_gbxs"); + std::cout << "\n--- create gridboxes ---\ninitialising\n"; const dualview_gbx gbxs(initialise_gbxs(gbxmaps, gbxic, totsupers)); diff --git a/libs/runcleo/createsupers.hpp b/libs/runcleo/createsupers.hpp index 9b6df36e..aa54f914 100644 --- a/libs/runcleo/createsupers.hpp +++ b/libs/runcleo/createsupers.hpp @@ -23,6 +23,7 @@ #define LIBS_RUNCLEO_CREATESUPERS_HPP_ #include +#include #include #include #include @@ -102,12 +103,17 @@ void print_supers(const viewd_constsupers supers); * ordered by the gridbox indexes and generated using a generator which uses * the initial conditions provided by the `SuperdropInitConds` type. * + * Kokkos::Profiling are null pointers unless a Kokkos profiler library has been + * exported to "KOKKOS_TOOLS_LIBS" prior to runtime so the lib gets dynamically loaded. + * * @tparam SuperdropInitConds The type of the super-droplets' initial conditions data. * @param sdic The instance of the super-droplets' initial conditions data. * @return A view of super-droplets in device memory. */ template viewd_supers create_supers(const SuperdropInitConds &sdic) { + Kokkos::Profiling::ScopedRegion region("init_supers"); + // Log message and create superdrops using the initial conditions std::cout << "\n--- create superdrops ---\ninitialising\n"; viewd_supers supers(initialise_supers(sdic)); diff --git a/libs/runcleo/runcleo.hpp b/libs/runcleo/runcleo.hpp index 19c021aa..41328096 100644 --- a/libs/runcleo/runcleo.hpp +++ b/libs/runcleo/runcleo.hpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -212,6 +213,9 @@ class RunCLEO { * * This function runs SDM on both host and device from `t_mdl` to `t_next`. * + * Kokkos::Profiling are null pointers unless a Kokkos profiler library has been + * exported to "KOKKOS_TOOLS_LIBS" prior to runtime so the lib gets dynamically loaded. + * * @param t_mdl Current timestep of the coupled model. * @param t_next Next timestep of the coupled model. * @param gbxs DualView of gridboxes. @@ -219,6 +223,8 @@ class RunCLEO { */ void sdm_step(const unsigned int t_mdl, unsigned int t_next, dualview_gbx gbxs, const viewd_supers totsupers) const { + Kokkos::Profiling::ScopedRegion region("timestep_sdm"); + gbxs.sync_device(); // get device up to date with host sdm.run_step(t_mdl, t_next, gbxs.view_device(), totsupers); gbxs.modify_device(); // mark device view of gbxs as modified @@ -229,10 +235,15 @@ class RunCLEO { * * This function runs the Coupled Dynamics on host from t_mdl to t_next. * + * Kokkos::Profiling are null pointers unless a Kokkos profiler library has been + * exported to "KOKKOS_TOOLS_LIBS" prior to runtime so the lib gets dynamically loaded. + * * @param t_mdl Current timestep of the coupled model. * @param t_next Next timestep of the coupled model. */ void coupldyn_step(const unsigned int t_mdl, const unsigned int t_next) const { + Kokkos::Profiling::ScopedRegion region("timestep_coupldyn"); + coupldyn.run_step(t_mdl, t_next); } @@ -287,21 +298,29 @@ class RunCLEO { * Creates runtime objects, gridboxes, superdrops and random number generators * using initial conditions, then prepares and performs CLEO timestepping. * + * Kokkos::Profiling are null pointers unless a Kokkos profiler library has been + * exported to "KOKKOS_TOOLS_LIBS" prior to runtime so the lib gets dynamically loaded. + * * @param initconds InitialConditions object containing initial conditions. * @param t_end End time for timestepping. * @return 0 on success. */ int operator()(const InitialConditions auto &initconds, const unsigned int t_end) const { - // create runtime objects - viewd_supers totsupers(create_supers(initconds.initsupers)); - dualview_gbx gbxs(create_gbxs(sdm.gbxmaps, initconds.initgbxs, totsupers)); + Kokkos::Profiling::pushRegion("runcleo"); - // prepare CLEO for timestepping + // create runtime objects and prepare CLEO for timestepping + Kokkos::Profiling::pushRegion("init"); + auto totsupers = create_supers(initconds.initsupers); + auto gbxs = create_gbxs(sdm.gbxmaps, initconds.initgbxs, totsupers); prepare_to_timestep(gbxs); + Kokkos::Profiling::popRegion(); // do timestepping from t=0 to t=t_end + Kokkos::Profiling::pushRegion("timestep"); timestep_cleo(t_end, gbxs, totsupers); + Kokkos::Profiling::popRegion(); + Kokkos::Profiling::popRegion(); return 0; } }; diff --git a/libs/runcleo/sdmmethods.hpp b/libs/runcleo/sdmmethods.hpp index 8237fdce..ec3a58fe 100644 --- a/libs/runcleo/sdmmethods.hpp +++ b/libs/runcleo/sdmmethods.hpp @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -89,6 +90,9 @@ class SDMMethods { * `movesupers` is an instance of the MoveSupersInDomain templated type with a certain * instance of a type of GridboxMaps, super-droplets' Motion and boundary conditions. * + * Kokkos::Profiling are null pointers unless a Kokkos profiler library has been + * exported to "KOKKOS_TOOLS_LIBS" prior to runtime so the lib gets dynamically loaded. + * * @param t_sdm Current timestep for SDM. * @param d_gbxs View of gridboxes on device. * @param totsupers View of all superdrops (both in and out of bounds of domain). @@ -96,6 +100,8 @@ class SDMMethods { */ void superdrops_movement(const unsigned int t_sdm, viewd_gbx d_gbxs, const viewd_supers totsupers, const SDMMonitor auto mo) const { + Kokkos::Profiling::ScopedRegion region("timestep_sdm_movement"); + movesupers.run_step(t_sdm, gbxmaps, d_gbxs, totsupers, mo); } @@ -122,9 +128,12 @@ class SDMMethods { * @brief run SDM microphysics for each gridbox (using sub-timestepping routine). * * This function runs SDM microphysics for each gridbox using a sub-timestepping routine. - * Kokkos::parallel_for is nested parallelism within parallelised loop over gridboxes, + * Kokkos::parallel_for is nested parallelism within parallelised loop over gridboxes, * serial equivalent is simply: `for (size_t ii(0); ii < ngbxs; ++ii) { [...] }` * + * Kokkos::Profiling are null pointers unless a Kokkos profiler library has been + * exported to "KOKKOS_TOOLS_LIBS" prior to runtime so the lib gets dynamically loaded. + * * @param t_sdm Current timestep for SDM. * @param t_next Next timestep for SDM. * @param d_gbxs View of gridboxes on device. @@ -133,6 +142,8 @@ class SDMMethods { template void operator()(const unsigned int t_sdm, const unsigned int t_next, const viewd_gbx d_gbxs, const SDMMo mo) const { + Kokkos::Profiling::ScopedRegion region("timestep_sdm_microphysics"); + // TODO(all) use scratch space for parallel region const size_t ngbxs(d_gbxs.extent(0)); Kokkos::parallel_for(