Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable GPU exection of atm_divergence_damping_3d via OpenACC #1237

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Enable GPU exection of atm_divergence_damping_3d via OpenACC #1237

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ad893a0
Initial OpenACC port of atm_divergence_damping_3d
abishekg7 Aug 7, 2024
d643283
Moving acc data movements of invariant fields to init and finalize
abishekg7 Dec 19, 2024
d092c2d
Removing async from parallel clause
abishekg7 Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions src/core_atmosphere/dynamics/mpas_atm_time_integration.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ subroutine mpas_atm_dynamics_init(domain)
real (kind=RKIND), dimension(:), pointer :: invAreaCell
integer, dimension(:), pointer :: bdyMaskCell
integer, dimension(:), pointer :: bdyMaskEdge
real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
real (kind=RKIND), dimension(:), pointer :: invDvEdge
real (kind=RKIND), dimension(:), pointer :: dcEdge
real (kind=RKIND), dimension(:), pointer :: invDcEdge
Expand Down Expand Up @@ -288,6 +289,9 @@ subroutine mpas_atm_dynamics_init(domain)
call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge)
!$acc enter data copyin(bdyMaskEdge)

call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge)
!$acc enter data copyin(specZoneMaskEdge)

call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge)
!$acc enter data copyin(invDvEdge)

Expand Down Expand Up @@ -383,6 +387,7 @@ subroutine mpas_atm_dynamics_finalize(domain)
real (kind=RKIND), dimension(:), pointer :: invAreaCell
integer, dimension(:), pointer :: bdyMaskCell
integer, dimension(:), pointer :: bdyMaskEdge
real (kind=RKIND), dimension(:), pointer :: specZoneMaskEdge
real (kind=RKIND), dimension(:), pointer :: invDvEdge
real (kind=RKIND), dimension(:), pointer :: dcEdge
real (kind=RKIND), dimension(:), pointer :: invDcEdge
Expand Down Expand Up @@ -459,6 +464,9 @@ subroutine mpas_atm_dynamics_finalize(domain)
call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge)
!$acc exit data delete(bdyMaskEdge)

call mpas_pool_get_array(mesh, 'specZoneMaskEdge', specZoneMaskEdge)
!$acc exit data delete(specZoneMaskEdge)

call mpas_pool_get_array(mesh, 'invDvEdge', invDvEdge)
!$acc exit data delete(invDvEdge)

Expand Down Expand Up @@ -2599,6 +2607,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
rdts = 1.0_RKIND / dts
coef_divdamp = 2.0_RKIND * smdiv * config_len_disp * rdts

MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]')
!$acc data present(specZoneMaskEdge, cellsOnEdge)
!$acc enter data copyin(ru_p, rtheta_pp, rtheta_pp_old, theta_m)
MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]')

!$acc parallel
!$acc loop gang worker
do iEdge=edgeStart,edgeEnd ! MGD do we really just need edges touching owned cells?

cell1 = cellsOnEdge(1,iEdge)
Expand All @@ -2608,6 +2623,7 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
if (cell1 <= nCellsSolve .or. cell2 <= nCellsSolve ) then

!DIR$ IVDEP
!$acc loop vector
do k=1,nVertLevels

!! unscaled 3d divergence damping
Expand All @@ -2625,6 +2641,13 @@ subroutine atm_divergence_damping_3d( state, diag, mesh, configs, dts, edgeStart
end do
end if ! edges for block-owned cells
end do ! end loop over edges
!$acc end parallel

MPAS_ACC_TIMER_START('atm_divergence_damping_3d [ACC_data_xfer]')
!$acc exit data copyout(ru_p) delete(rtheta_pp, rtheta_pp_old, theta_m)
!$acc end data
MPAS_ACC_TIMER_STOP('atm_divergence_damping_3d [ACC_data_xfer]')


end subroutine atm_divergence_damping_3d

Expand Down