diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 6a8005c8b2..50a5f3cdb8 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -212,6 +212,7 @@ subroutine mpas_atm_dynamics_init(domain) real (kind=RKIND), dimension(:), pointer :: invAreaCell integer, dimension(:), pointer :: bdyMaskCell integer, dimension(:), pointer :: bdyMaskEdge + real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge real (kind=RKIND), dimension(:), pointer :: invDvEdge real (kind=RKIND), dimension(:), pointer :: dcEdge real (kind=RKIND), dimension(:), pointer :: invDcEdge @@ -288,6 +289,9 @@ subroutine mpas_atm_dynamics_init(domain) call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge) !$acc enter data copyin(bdyMaskEdge) + call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge) + !$acc enter data copyin(specZoneMaskEdge) + call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge) !$acc enter data copyin(invDvEdge) @@ -383,6 +387,7 @@ subroutine mpas_atm_dynamics_finalize(domain) real (kind=RKIND), dimension(:), pointer :: invAreaCell integer, dimension(:), pointer :: bdyMaskCell integer, dimension(:), pointer :: bdyMaskEdge + real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge real (kind=RKIND), dimension(:), pointer :: invDvEdge real (kind=RKIND), dimension(:), pointer :: dcEdge real (kind=RKIND), dimension(:), pointer :: invDcEdge @@ -459,6 +464,9 @@ subroutine mpas_atm_dynamics_finalize(domain) call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge) !$acc exit data delete(bdyMaskEdge) + call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge) + !$acc exit data delete(specZoneMaskEdge) + call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge) !$acc exit data delete(invDvEdge) @@ -2599,6 +2607,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart rdts = 1.0_RKIND / dts coef_divdamp = 2.0_RKIND * smdiv * config_len_disp * rdts + MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]') + !$acc data present(specZoneMaskEdge, cellsOnEdge) + !$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, theta_m) + MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]') + + !$acc parallel + !$acc loop gang worker do iEdge=edgeStart,edgeEnd ! MGD do we really just need edges touching owned cells? cell1 = cellsOnEdge(1,iEdge) @@ -2608,6 +2623,7 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart if (cell1 <= nCellsSolve .or. cell2 <= nCellsSolve ) then !DIR$ IVDEP + !$acc loop vector do k=1,nVertLevels !! unscaled 3d divergence damping @@ -2625,6 +2641,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart end do end if ! edges for block-owned cells end do ! end loop over edges + !$acc end parallel + + MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]') + !$acc exit data copyout(ru_p) delete(rtheta_pp, rtheta_pp_old, theta_m) + !$acc end data + MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]') + end subroutine atm_divergence_damping_3d