diff --git a/CMakeLists.txt b/CMakeLists.txt index afd224ed..8a4aa89e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,8 @@ option(use_WRTCOMP "Enable compiler definition -Duse_WRTCOMP" OFF) option(INTERNAL_FILE_NML "Enable compiler definition -DINTERNAL_FILE_NML" ON) option(ENABLE_QUAD_PRECISION "Enable compiler definition -DENABLE_QUAD_PRECISION" ON) option(HYDRO "Enable compiler definition -DHYDRO" OFF) +option(ENABLE_RRFS_WAR "Enable independent parallel I/O when io_layout=1,1 -DENABLE_RRFS_WAR" OFF) +option(ENABLE_PARALLELRESTART "Enable collective parallel reads -DENABLE_PARALLELRESTART" ON) find_package(MPI REQUIRED) if(OPENMP) @@ -167,6 +169,14 @@ if(ENABLE_QUAD_PRECISION) list(APPEND fv3_defs ENABLE_QUAD_PRECISION) endif() +if(ENABLE_PARALLELRESTART) + list(APPEND fv3_defs ENABLE_PARALLELRESTART) +endif() + +if(ENABLE_RRFS_WAR) + list(APPEND fv3_defs ENABLE_RRFS_WAR) +endif() + if(OPENMP) list(APPEND fv3_defs OPENMP) endif() diff --git a/tools/external_ic.F90 b/tools/external_ic.F90 index 157f0467..14ff72ca 100644 --- a/tools/external_ic.F90 +++ b/tools/external_ic.F90 @@ -154,7 +154,17 @@ module external_ic_mod use mpp_mod, only: mpp_error, FATAL, NOTE, mpp_pe, mpp_root_pe use mpp_mod, only: stdlog, input_nml_file, mpp_npes, mpp_get_current_pelist use mpp_parameter_mod, only: AGRID_PARAM=>AGRID - use mpp_domains_mod, only: mpp_get_tile_id, domain2d, mpp_update_domains, NORTH, EAST +#ifdef ENABLE_PARALLELRESTART + use mpp_domains_mod, only: mpp_get_tile_id, domain2d, mpp_update_domains, & + NORTH, EAST, mpp_get_domain_tile_commid, & + mpp_get_io_domain_layout, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#else + use mpp_domains_mod, only: mpp_get_tile_id, domain2d, mpp_update_domains, & + NORTH, EAST, & + mpp_get_io_domain_layout, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#endif use tracer_manager_mod, only: get_tracer_names, get_number_tracers, get_tracer_index use tracer_manager_mod, only: set_tracer_profile use field_manager_mod, only: MODEL_ATMOS @@ -875,6 +885,8 @@ subroutine get_nggps_ic (Atm) contains subroutine read_gfs_ic() + integer :: read_layout(2) + type(domain2D) :: domain_for_read ! !--- read in ak and bk from the gfs control file using fms_io read_data --- ! @@ -916,7 +928,17 @@ subroutine read_gfs_ic() dim_names_3d4(1) = "levp" ! surface pressure (Pa) +#ifdef ENABLE_PARALLELRESTART + call mpp_get_layout(Atm%domain,read_layout) + call mpp_copy_domain(Atm%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + + GFS_restart%use_collective = .true. + GFS_restart%tile_comm = mpp_get_domain_tile_commid(Atm%domain) + if( open_file(GFS_restart, fn_gfs_ics, "read", domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) ) then +#else if( open_file(GFS_restart, fn_gfs_ics, "read", Atm%domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) ) then +#endif call register_axis(GFS_restart, "lat", "y") call register_axis(GFS_restart, "lon", "x") call register_axis(GFS_restart, "lonp", "x", domain_position=east) diff --git a/tools/fv_io.F90 b/tools/fv_io.F90 index 74d6e1ce..e050db22 100644 --- a/tools/fv_io.F90 +++ b/tools/fv_io.F90 @@ -94,11 +94,19 @@ module fv_io_mod variable_exists, read_data, set_filename_appendix, get_dimension_size use mpp_mod, only: mpp_error, FATAL, NOTE, WARNING, mpp_root_pe, & mpp_sync, mpp_pe, mpp_declare_pelist, mpp_get_current_pelist, & +#ifdef ENABLE_PARALLELRESTART + mpp_npes, MPP_COMM_NULL +#else mpp_npes +#endif use mpp_domains_mod, only: domain2d, EAST, WEST, NORTH, CENTER, SOUTH, CORNER, & mpp_get_compute_domain, mpp_get_data_domain, & - mpp_get_layout, mpp_get_ntile_count, & + mpp_get_layout, mpp_get_ntile_count, mpp_copy_domain, & +#ifdef ENABLE_PARALLELRESTART + mpp_get_global_domain, mpp_get_domain_tile_commid, mpp_define_io_domain +#else mpp_get_global_domain +#endif use tracer_manager_mod, only: tr_get_tracer_names=>get_tracer_names, & get_tracer_names, get_number_tracers, & set_tracer_profile, & @@ -438,6 +446,8 @@ subroutine fv_io_read_restart(fv_domain,Atm) character(len=20) :: suffix character(len=1) :: tile_num integer, allocatable, dimension(:) :: pes !< Array of the pes in the current pelist + type(domain2D) :: domain_for_read + integer :: read_layout(2), layout(2) allocate(pes(mpp_npes())) call mpp_get_current_pelist(pes) @@ -451,7 +461,7 @@ subroutine fv_io_read_restart(fv_domain,Atm) call close_file(Atm(1)%Fv_restart) Atm(1)%Fv_restart_is_open = .false. endif - deallocate(pes) + !deallocate(pes) if (Atm(1)%flagstruct%external_eta) then call set_external_eta(Atm(1)%ak, Atm(1)%bk, Atm(1)%ptop, Atm(1)%ks) @@ -469,17 +479,38 @@ subroutine fv_io_read_restart(fv_domain,Atm) endif fname = 'INPUT/fv_core.res'//trim(suffix)//'.nc' +#ifdef ENABLE_PARALLELRESTART + Atm(1)%Fv_restart_tile%use_collective = .true. + + call mpp_get_layout(Atm(1)%domain,read_layout) + call mpp_copy_domain(Atm(1)%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + Atm(1)%Fv_restart_tile%tile_comm = mpp_get_domain_tile_commid(Atm(1)%domain) + Atm(1)%Fv_restart_tile_is_open = open_file(Atm(1)%Fv_restart_tile, fname, "read", domain_for_read, is_restart=.true.) +#else Atm(1)%Fv_restart_tile_is_open = open_file(Atm(1)%Fv_restart_tile, fname, "read", fv_domain, is_restart=.true.) +#endif if (Atm(1)%Fv_restart_tile_is_open) then call fv_io_register_restart(Atm(1)) call read_restart(Atm(1)%Fv_restart_tile, ignore_checksum=Atm(1)%flagstruct%ignore_rst_cksum) call close_file(Atm(1)%Fv_restart_tile) Atm(1)%Fv_restart_tile_is_open = .false. endif +#ifdef ENABLE_PARALLELRESTART + ! Disable use_collective so the reuse of this Fv_restart_tile during checkpointing doesn't fail + Atm(1)%Fv_restart_tile%use_collective = .false. + Atm(1)%Fv_restart_tile%tile_comm = MPP_COMM_NULL +#endif !--- restore data for fv_tracer - if it exists fname = 'INPUT/fv_tracer.res'//trim(suffix)//'.nc' +#ifdef ENABLE_PARALLELRESTART + Atm(1)%Tra_restart%use_collective = .true. + Atm(1)%Tra_restart%tile_comm = mpp_get_domain_tile_commid(Atm(1)%domain) + Atm(1)%Tra_restart_is_open = open_file(Atm(1)%Tra_restart, fname, "read", domain_for_read, is_restart=.true.) +#else Atm(1)%Tra_restart_is_open = open_file(Atm(1)%Tra_restart, fname, "read", fv_domain, is_restart=.true.) +#endif if (Atm(1)%Tra_restart_is_open) then call fv_io_register_restart(Atm(1)) call read_restart(Atm(1)%Tra_restart, ignore_checksum=Atm(1)%flagstruct%ignore_rst_cksum) @@ -488,6 +519,11 @@ subroutine fv_io_read_restart(fv_domain,Atm) else call mpp_error(NOTE,'==> Warning from fv_read_restart: Expected file '//trim(fname)//' does not exist') endif +#ifdef ENABLE_PARALLELRESTART + ! Disable use_collective so the reuse of this Tra_restart during checkpointing doesn't fail + Atm(1)%Tra_restart%use_collective = .false. + Atm(1)%Fv_restart_tile%tile_comm = MPP_COMM_NULL +#endif !--- restore data for surface winds - if it exists fname = 'INPUT/fv_srf_wnd.res'//trim(suffix)//'.nc' @@ -528,6 +564,7 @@ subroutine fv_io_read_restart(fv_domain,Atm) endif endif + deallocate(pes) return end subroutine fv_io_read_restart diff --git a/tools/fv_mp_mod.F90 b/tools/fv_mp_mod.F90 index 4f2c0f2c..bf6b1ede 100644 --- a/tools/fv_mp_mod.F90 +++ b/tools/fv_mp_mod.F90 @@ -658,12 +658,16 @@ subroutine domain_decomp(grid_num,npx,npy,nregions,grid_type,nested,layout,io_la !--- if io_layout\=(1,1) then read io_layout=io_layout (no change) l_layout = mpp_get_io_domain_layout(domain) call mpp_copy_domain(domain, domain_for_read) +#ifdef ENABLE_RRFS_WAR if (ALL(l_layout == 1)) then call mpp_get_layout(domain, l_layout) call mpp_define_io_domain(domain_for_read, l_layout) else call mpp_define_io_domain(domain_for_read, l_layout) endif +#else + call mpp_define_io_domain(domain_for_read, l_layout) +#endif endif deallocate(pe_start,pe_end)