add initial working dummy MPI statements to SCM code

NCAR · May 28, 2024 · 37f6272 · 37f6272
1 parent 0cd476c
commit 37f6272
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 7 deletions.
diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt
@@ -105,6 +105,9 @@ ADD_DEFINITIONS(-DCCPP)
 # Add host-model specific preprocessor flag (needed for some physics schemes)
 ADD_DEFINITIONS(-DSCM)
 
+ADD_DEFINITIONS(-DMPI)
+
+
 #------------------------------------------------------------------------------
 # Add model-specific flags for C/C++/Fortran preprocessor
 ADD_DEFINITIONS(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM)
@@ -257,7 +260,7 @@ endif()
 # How about using proper compile targets etc?
 #------------------------------------------------------------------------------
 # Configure sources
-set(MPI ON)
+
 ADD_SUBDIRECTORY(${CCPP_FRAMEWORK_SRC} ${CMAKE_BINARY_DIR}/ccpp/framework)
 ADD_SUBDIRECTORY(${CCPP_PHYSICS_SRC} ${CMAKE_BINARY_DIR}/ccpp/physics)
 ADD_DEPENDENCIES(ccpp_physics ccpp_framework)

diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py
@@ -720,7 +720,7 @@ def launch_executable(use_gdb, gdb, ignore_error = False):
     if use_gdb:
         cmd = '(cd {scm_run} && {gdb} {executable})'.format(scm_run=SCM_RUN, gdb=gdb, executable=EXECUTABLE)
     else:
-        cmd = '(cd {scm_run} && time {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE)
+        cmd = '(cd {scm_run} && time srun -A gmtb -n 1 {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE)
     logging.info('Passing control to "{0}"'.format(cmd))
     time.sleep(1)
     # This will abort in 'execute' in the event of an error if ignore_error = False

diff --git a/scm/src/scm.F90 b/scm/src/scm.F90
@@ -15,7 +15,8 @@ subroutine scm_main_sub()
   use scm_time_integration
   use scm_output
   use scm_type_defs
-
+  use mpi_f08
+
   use :: ccpp_static_api,                      &
          only: ccpp_physics_init,              &
                ccpp_physics_timestep_init,     &
@@ -30,6 +31,8 @@ subroutine scm_main_sub()
   type(scm_input_type), target :: scm_input_instance
   type(scm_reference_type), target :: scm_reference
 
+  type(MPI_Comm)           :: fcst_mpi_comm
+
   integer      :: i, j, kdt_rad, idtend, itrac
   real(kind=8) :: rinc(5) !(DAYS, HOURS, MINUTES, SECONDS, MILLISECONDS)
   integer      :: jdat(1:8)
@@ -39,6 +42,11 @@ subroutine scm_main_sub()
   character(len=16) :: logfile_name
   logical                           :: in_spinup
 
+#ifdef MPI
+  call MPI_INIT(ierr)
+  fcst_mpi_comm = MPI_COMM_WORLD
+#endif  
+
   call get_config_nml(scm_state)
 
   select case(scm_state%input_type)
@@ -94,6 +102,7 @@ subroutine scm_main_sub()
     open(unit=physics%Init_parm%logunit, file=trim(scm_state%output_dir)//'/'//logfile_name, action='write', status='replace')
   end if
 
+  physics%Init_parm%fcst_mpi_comm   =  fcst_mpi_comm
   physics%Init_parm%levs = scm_state%n_levels
   physics%Init_parm%bdat(1) = scm_state%init_year
   physics%Init_parm%bdat(2) = scm_state%init_month
@@ -120,7 +129,7 @@ subroutine scm_main_sub()
   call GFS_suite_setup(physics%Model, physics%Statein, physics%Stateout,           &
                        physics%Sfcprop, physics%Coupling, physics%Grid,            &
                        physics%Tbd, physics%Cldprop, physics%Radtend,              &
-                       physics%Diag, physics%Interstitial, 0, 1, 1,                &
+                       physics%Diag, physics%Interstitial, 1, 1,                   &
                        physics%Init_parm, scm_state%n_cols, scm_state%lon,         &
                        scm_state%lat, scm_state%area)
 
@@ -429,5 +438,10 @@ end module scm_main
 !! subroutine \ref scm_main_sub above.
 program scm
   use scm_main
+
+#ifdef MPI
+  use mpi_f08
+#endif
+
   call scm_main_sub()
 end program scm
diff --git a/scm/src/scm_setup.F90 b/scm/src/scm_setup.F90
@@ -279,7 +279,7 @@ end subroutine patch_in_ref
 !--------------
 subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop,                   &
                             Coupling, Grid, Tbd, Cldprop, Radtend, Diag,         &
-                            Interstitial, communicator, ntasks, nthreads,        &
+                            Interstitial, ntasks, nthreads,                      &
                             Init_parm, n_cols, lon, lat, area)
 
   use machine,             only: kind_phys
@@ -311,7 +311,6 @@ subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop,
   type(GFS_interstitial_type),               intent(inout) :: Interstitial
   type(GFS_init_type),                       intent(in)    :: Init_parm
 
-  integer,                  intent(in)    :: communicator
   integer,                  intent(in)    :: ntasks, nthreads, n_cols
 
   real(kind=dp), dimension(n_cols), intent(in) :: lon, lat, area
@@ -334,7 +333,7 @@ subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop,
                    Init_parm%input_nml_file, Init_parm%tile_num,   &
                    Init_parm%blksz, Init_parm%ak, Init_parm%bk,    &
                    Init_parm%restart, Init_parm%hydrostatic,       &
-                   communicator, ntasks, nthreads)
+                   Init_parm%fcst_mpi_comm, ntasks, nthreads)
 
   !--- initialize DDTs