From ad893a072b804b2f3d2f522738fe89168a88939d Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Wed, 7 Aug 2024 10:52:14 -0600 Subject: [PATCH 1/3] Initial OpenACC port of atm_divergence_damping_3d --- .../dynamics/mpas_atm_time_integration.F | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 6a8005c8b2..f812441cfe 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -2599,6 +2599,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart rdts = 1.0_RKIND / dts coef_divdamp = 2.0_RKIND * smdiv * config_len_disp * rdts + MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]') + !$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, specZoneMaskEdge, & + !$acc theta_m, cellsOnEdge) + MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]') + + !$acc parallel async + !$acc loop gang worker do iEdge=edgeStart,edgeEnd ! MGD do we really just need edges touching owned cells? cell1 = cellsOnEdge(1,iEdge) @@ -2608,6 +2615,7 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart if (cell1 <= nCellsSolve .or. cell2 <= nCellsSolve ) then !DIR$ IVDEP + !$acc loop vector do k=1,nVertLevels !! unscaled 3d divergence damping @@ -2625,6 +2633,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart end do end if ! edges for block-owned cells end do ! end loop over edges + !$acc end parallel + + MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]') + !$acc exit data copyout(ru_p) delete(rtheta_pp, rtheta_pp_old, & + !$acc specZoneMaskEdge, theta_m, cellsOnEdge) + MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]') + end subroutine atm_divergence_damping_3d From d64328346c8e9e9396b1cca96d3f4aac6c3deae4 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 19 Dec 2024 16:01:04 -0700 Subject: [PATCH 2/3] Moving acc data movements of invariant fields to init and finalize --- .../dynamics/mpas_atm_time_integration.F | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index f812441cfe..a120dc4ba0 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -212,6 +212,7 @@ subroutine mpas_atm_dynamics_init(domain) real (kind=RKIND), dimension(:), pointer :: invAreaCell integer, dimension(:), pointer :: bdyMaskCell integer, dimension(:), pointer :: bdyMaskEdge + real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge real (kind=RKIND), dimension(:), pointer :: invDvEdge real (kind=RKIND), dimension(:), pointer :: dcEdge real (kind=RKIND), dimension(:), pointer :: invDcEdge @@ -288,6 +289,9 @@ subroutine mpas_atm_dynamics_init(domain) call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge) !$acc enter data copyin(bdyMaskEdge) + call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge) + !$acc enter data copyin(specZoneMaskEdge) + call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge) !$acc enter data copyin(invDvEdge) @@ -383,6 +387,7 @@ subroutine mpas_atm_dynamics_finalize(domain) real (kind=RKIND), dimension(:), pointer :: invAreaCell integer, dimension(:), pointer :: bdyMaskCell integer, dimension(:), pointer :: bdyMaskEdge + real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge real (kind=RKIND), dimension(:), pointer :: invDvEdge real (kind=RKIND), dimension(:), pointer :: dcEdge real (kind=RKIND), dimension(:), pointer :: invDcEdge @@ -459,6 +464,9 @@ subroutine mpas_atm_dynamics_finalize(domain) call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge) !$acc exit data delete(bdyMaskEdge) + call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge) + !$acc exit data delete(specZoneMaskEdge) + call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge) !$acc exit data delete(invDvEdge) @@ -2600,8 +2608,8 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart coef_divdamp = 2.0_RKIND * smdiv * config_len_disp * rdts MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]') - !$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, specZoneMaskEdge, & - !$acc theta_m, cellsOnEdge) + !$acc data present(specZoneMaskEdge, cellsOnEdge) + !$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, theta_m) MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]') !$acc parallel async @@ -2636,8 +2644,8 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart !$acc end parallel MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]') - !$acc exit data copyout(ru_p) delete(rtheta_pp, rtheta_pp_old, & - !$acc specZoneMaskEdge, theta_m, cellsOnEdge) + !$acc exit data copyout(ru_p) delete(rtheta_pp, rtheta_pp_old, theta_m) + !$acc end data MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]') From d092c2d76379bb6c0d3e8cf2b8e65173cf3184fe Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Thu, 19 Dec 2024 16:03:30 -0700 Subject: [PATCH 3/3] Removing async from parallel clause --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index a120dc4ba0..50a5f3cdb8 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -2612,7 +2612,7 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart !$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, theta_m) MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]') - !$acc parallel async + !$acc parallel !$acc loop gang worker do iEdge=edgeStart,edgeEnd ! MGD do we really just need edges touching owned cells?