Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sixth reconciliation PR from production/RRFS.v1 #365

Open
wants to merge 1 commit into
base: dev/emc
Choose a base branch
from
Open

Sixth reconciliation PR from production/RRFS.v1 #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ option(use_WRTCOMP "Enable compiler definition -Duse_WRTCOMP" OFF)
option(INTERNAL_FILE_NML "Enable compiler definition -DINTERNAL_FILE_NML" ON)
option(ENABLE_QUAD_PRECISION "Enable compiler definition -DENABLE_QUAD_PRECISION" ON)
option(HYDRO "Enable compiler definition -DHYDRO" OFF)
option(ENABLE_RRFS_WAR "Enable independent parallel I/O when io_layout=1,1 -DENABLE_RRFS_WAR" OFF)
option(ENABLE_PARALLELRESTART "Enable collective parallel reads -DENABLE_PARALLELRESTART" ON)

find_package(MPI REQUIRED)
if(OPENMP)
Expand Down Expand Up @@ -167,6 +169,14 @@ if(ENABLE_QUAD_PRECISION)
list(APPEND fv3_defs ENABLE_QUAD_PRECISION)
endif()

if(ENABLE_PARALLELRESTART)
list(APPEND fv3_defs ENABLE_PARALLELRESTART)
endif()

if(ENABLE_RRFS_WAR)
list(APPEND fv3_defs ENABLE_RRFS_WAR)
endif()

if(OPENMP)
list(APPEND fv3_defs OPENMP)
endif()
Expand Down
24 changes: 23 additions & 1 deletion tools/external_ic.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,17 @@ module external_ic_mod
use mpp_mod, only: mpp_error, FATAL, NOTE, mpp_pe, mpp_root_pe
use mpp_mod, only: stdlog, input_nml_file, mpp_npes, mpp_get_current_pelist
use mpp_parameter_mod, only: AGRID_PARAM=>AGRID
use mpp_domains_mod, only: mpp_get_tile_id, domain2d, mpp_update_domains, NORTH, EAST
#ifdef ENABLE_PARALLELRESTART
use mpp_domains_mod, only: mpp_get_tile_id, domain2d, mpp_update_domains, &
NORTH, EAST, mpp_get_domain_tile_commid, &
mpp_get_io_domain_layout, mpp_copy_domain, &
mpp_define_io_domain, mpp_get_layout
#else
use mpp_domains_mod, only: mpp_get_tile_id, domain2d, mpp_update_domains, &
NORTH, EAST, &
mpp_get_io_domain_layout, mpp_copy_domain, &
mpp_define_io_domain, mpp_get_layout
#endif
use tracer_manager_mod, only: get_tracer_names, get_number_tracers, get_tracer_index
use tracer_manager_mod, only: set_tracer_profile
use field_manager_mod, only: MODEL_ATMOS
Expand Down Expand Up @@ -875,6 +885,8 @@ subroutine get_nggps_ic (Atm)
contains

subroutine read_gfs_ic()
integer :: read_layout(2)
type(domain2D) :: domain_for_read
!
!--- read in ak and bk from the gfs control file using fms_io read_data ---
!
Expand Down Expand Up @@ -916,7 +928,17 @@ subroutine read_gfs_ic()
dim_names_3d4(1) = "levp"

! surface pressure (Pa)
#ifdef ENABLE_PARALLELRESTART
call mpp_get_layout(Atm%domain,read_layout)
call mpp_copy_domain(Atm%domain, domain_for_read)
call mpp_define_io_domain(domain_for_read, read_layout)

GFS_restart%use_collective = .true.
GFS_restart%tile_comm = mpp_get_domain_tile_commid(Atm%domain)
if( open_file(GFS_restart, fn_gfs_ics, "read", domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) ) then
#else
if( open_file(GFS_restart, fn_gfs_ics, "read", Atm%domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) ) then
#endif
call register_axis(GFS_restart, "lat", "y")
call register_axis(GFS_restart, "lon", "x")
call register_axis(GFS_restart, "lonp", "x", domain_position=east)
Expand Down
41 changes: 39 additions & 2 deletions tools/fv_io.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,19 @@ module fv_io_mod
variable_exists, read_data, set_filename_appendix, get_dimension_size
use mpp_mod, only: mpp_error, FATAL, NOTE, WARNING, mpp_root_pe, &
mpp_sync, mpp_pe, mpp_declare_pelist, mpp_get_current_pelist, &
#ifdef ENABLE_PARALLELRESTART
mpp_npes, MPP_COMM_NULL
#else
mpp_npes
#endif
use mpp_domains_mod, only: domain2d, EAST, WEST, NORTH, CENTER, SOUTH, CORNER, &
mpp_get_compute_domain, mpp_get_data_domain, &
mpp_get_layout, mpp_get_ntile_count, &
mpp_get_layout, mpp_get_ntile_count, mpp_copy_domain, &
#ifdef ENABLE_PARALLELRESTART
mpp_get_global_domain, mpp_get_domain_tile_commid, mpp_define_io_domain
#else
mpp_get_global_domain
#endif
use tracer_manager_mod, only: tr_get_tracer_names=>get_tracer_names, &
get_tracer_names, get_number_tracers, &
set_tracer_profile, &
Expand Down Expand Up @@ -438,6 +446,8 @@ subroutine fv_io_read_restart(fv_domain,Atm)
character(len=20) :: suffix
character(len=1) :: tile_num
integer, allocatable, dimension(:) :: pes !< Array of the pes in the current pelist
type(domain2D) :: domain_for_read
integer :: read_layout(2), layout(2)

allocate(pes(mpp_npes()))
call mpp_get_current_pelist(pes)
Expand All @@ -451,7 +461,7 @@ subroutine fv_io_read_restart(fv_domain,Atm)
call close_file(Atm(1)%Fv_restart)
Atm(1)%Fv_restart_is_open = .false.
endif
deallocate(pes)
!deallocate(pes)

if (Atm(1)%flagstruct%external_eta) then
call set_external_eta(Atm(1)%ak, Atm(1)%bk, Atm(1)%ptop, Atm(1)%ks)
Expand All @@ -469,17 +479,38 @@ subroutine fv_io_read_restart(fv_domain,Atm)
endif

fname = 'INPUT/fv_core.res'//trim(suffix)//'.nc'
#ifdef ENABLE_PARALLELRESTART
Atm(1)%Fv_restart_tile%use_collective = .true.

call mpp_get_layout(Atm(1)%domain,read_layout)
call mpp_copy_domain(Atm(1)%domain, domain_for_read)
call mpp_define_io_domain(domain_for_read, read_layout)
Atm(1)%Fv_restart_tile%tile_comm = mpp_get_domain_tile_commid(Atm(1)%domain)
Atm(1)%Fv_restart_tile_is_open = open_file(Atm(1)%Fv_restart_tile, fname, "read", domain_for_read, is_restart=.true.)
#else
Atm(1)%Fv_restart_tile_is_open = open_file(Atm(1)%Fv_restart_tile, fname, "read", fv_domain, is_restart=.true.)
#endif
if (Atm(1)%Fv_restart_tile_is_open) then
call fv_io_register_restart(Atm(1))
call read_restart(Atm(1)%Fv_restart_tile, ignore_checksum=Atm(1)%flagstruct%ignore_rst_cksum)
call close_file(Atm(1)%Fv_restart_tile)
Atm(1)%Fv_restart_tile_is_open = .false.
endif
#ifdef ENABLE_PARALLELRESTART
! Disable use_collective so the reuse of this Fv_restart_tile during checkpointing doesn't fail
Atm(1)%Fv_restart_tile%use_collective = .false.
Atm(1)%Fv_restart_tile%tile_comm = MPP_COMM_NULL
#endif

!--- restore data for fv_tracer - if it exists
fname = 'INPUT/fv_tracer.res'//trim(suffix)//'.nc'
#ifdef ENABLE_PARALLELRESTART
Atm(1)%Tra_restart%use_collective = .true.
Atm(1)%Tra_restart%tile_comm = mpp_get_domain_tile_commid(Atm(1)%domain)
Atm(1)%Tra_restart_is_open = open_file(Atm(1)%Tra_restart, fname, "read", domain_for_read, is_restart=.true.)
#else
Atm(1)%Tra_restart_is_open = open_file(Atm(1)%Tra_restart, fname, "read", fv_domain, is_restart=.true.)
#endif
if (Atm(1)%Tra_restart_is_open) then
call fv_io_register_restart(Atm(1))
call read_restart(Atm(1)%Tra_restart, ignore_checksum=Atm(1)%flagstruct%ignore_rst_cksum)
Expand All @@ -488,6 +519,11 @@ subroutine fv_io_read_restart(fv_domain,Atm)
else
call mpp_error(NOTE,'==> Warning from fv_read_restart: Expected file '//trim(fname)//' does not exist')
endif
#ifdef ENABLE_PARALLELRESTART
! Disable use_collective so the reuse of this Tra_restart during checkpointing doesn't fail
Atm(1)%Tra_restart%use_collective = .false.
Atm(1)%Fv_restart_tile%tile_comm = MPP_COMM_NULL
#endif

!--- restore data for surface winds - if it exists
fname = 'INPUT/fv_srf_wnd.res'//trim(suffix)//'.nc'
Expand Down Expand Up @@ -528,6 +564,7 @@ subroutine fv_io_read_restart(fv_domain,Atm)
endif
endif

deallocate(pes)
return

end subroutine fv_io_read_restart
Expand Down
4 changes: 4 additions & 0 deletions tools/fv_mp_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,12 +658,16 @@ subroutine domain_decomp(grid_num,npx,npy,nregions,grid_type,nested,layout,io_la
!--- if io_layout\=(1,1) then read io_layout=io_layout (no change)
l_layout = mpp_get_io_domain_layout(domain)
call mpp_copy_domain(domain, domain_for_read)
#ifdef ENABLE_RRFS_WAR
if (ALL(l_layout == 1)) then
call mpp_get_layout(domain, l_layout)
call mpp_define_io_domain(domain_for_read, l_layout)
else
call mpp_define_io_domain(domain_for_read, l_layout)
endif
#else
call mpp_define_io_domain(domain_for_read, l_layout)
#endif
endif

deallocate(pe_start,pe_end)
Expand Down