Make cosmo job run

cases/cosmo-ghg-11km-test/config.py

@@ -129,7 +129,6 @@
 
 # Domain decomposition
 int2lm_nodes = 2
-int2lm_ntasks_per_node = 12
 int2lm_np_x = 8
 int2lm_np_y = 3
 int2lm_np_tot = int2lm_np_x * int2lm_np_y

cases/cosmo-ghg-11km-test/config.yaml

@@ -2,6 +2,7 @@
 
 model: cosmo-ghg
 constraint: gpu
+ntasks_per_node: 12
 restart_step: 12
 variant: spinup
 spinup: 6
@@ -53,7 +54,6 @@ int2lm:
     compute_queue: normal
     walltime: 01:00:00
     nodes: 2
-    ntasks_per_node: 12
     np_x: 8
     np_y: 3
 
@@ -73,6 +73,7 @@ cosmo:
     binary_file: ./src/cosmo-ghg/cosmo/ACC/cosmo_gpu
     namelist_prefix: cosmo_INPUT_
     runjob_filename: cosmo_runjob.cfg
+    compute_queue: normal
     walltime: "00:30:00"
     np_x: 6
     np_y: 4

cases/cosmo-ghg-11km-test/cosmo_INPUT_GHG.cfg

@@ -2,11 +2,11 @@
   in_tracers            = {cfg.in_tracers},
   tracer_start          = {cfg.tracer_start},
   iemiss_interp         = 1,
-  gridded_emissions_nc  = '../input/oem/{cfg.oem_gridded_emissions_nc}',
-  vertical_profile_nc   = '../input/oem/{cfg.oem_vertical_profiles_nc}',
-  hour_of_day_nc        = '../input/oem/{cfg.oem_hourofday_nc}',
-  day_of_week_nc        = '../input/oem/{cfg.oem_dayofweek_nc}',
-  month_of_year_nc      = '../input/oem/{cfg.oem_monthofyear_nc}',
+  gridded_emissions_nc  = '../input/oem/{gridded_emissions_nc}',
+  vertical_profile_nc   = '../input/oem/{vertical_profiles_nc}',
+  hour_of_day_nc        = '../input/oem/{hourofday_nc}',
+  day_of_week_nc        = '../input/oem/{dayofweek_nc}',
+  month_of_year_nc      = '../input/oem/{monthofyear_nc}',
   modis_reflectances_nc = '../input/vprm/modis.nc',
   veg_class_frac_nc     = '../input/vprm/vegetation.nc',
   octe_maps_nc          = '../input/octe/maps.nc',

cases/cosmo-ghg-11km-test/cosmo_INPUT_ORG.cfg

@@ -15,8 +15,8 @@
   hstart        = {cfg.hstart},
   hstop         = {cfg.hstop},   
   ydate_ini     = '{cfg.inidate_yyyymmddhh}',
-  nprocx        = {cfg.cosmo_np_x},
-  nprocy        = {cfg.cosmo_np_y},
+  nprocx        = {np_x},
+  nprocy        = {np_y},
   nprocio       = 0,
   nproma        = -1,
   hincmxt       = 1.0,
@@ -30,7 +30,7 @@
   ldfi          = .FALSE.,
   ldump_ascii   = .FALSE.,
   ncomm_type    = 1,        ! (OF: we use 1 operationally since 2 was causing trouble with memory consumption, leak in the MPI library)
-  num_asynio_comm = {cfg.cosmo_np_io},
+  num_asynio_comm = {np_io},
   num_iope_percomm = 0,
   idbg_level    = 8,
   nboundlines   = 3,

cases/cosmo-ghg-11km-test/cosmo_runjob.cfg

@@ -1,12 +1,12 @@
 #!/bin/bash -l
 #SBATCH --job-name="cosmo_{cfg.inidate_yyyymmddhh}_{cfg.forecasttime}"
 #SBATCH --account={cfg.compute_account}
-#SBATCH --time={cfg.cosmo_walltime}
-#SBATCH --nodes={cfg.cosmo_np_tot}
+#SBATCH --time={walltime}
+#SBATCH --nodes={np_tot}
 #SBATCH --ntasks-per-core=1
 #SBATCH --ntasks-per-node={cfg.ntasks_per_node}
 #SBATCH --cpus-per-task=1
-#SBATCH --partition={cfg.compute_queue}
+#SBATCH --partition={compute_queue}
 #SBATCH --constraint={cfg.constraint}
 #SBATCH --hint=nomultithread
 #SBATCH --output={logfile}
@@ -34,7 +34,7 @@ echo "============== StartTime: `date +%s` s"
 echo "============== StartTime: `date`"
 echo "====================================================="
 
-srun -u ./{cfg.execname} >> {logfile} 2>&1
+srun -u ./{execname} >> {logfile} 2>&1
 pid=$?
 
 echo "====================================================="

cases/cosmo-ghg-11km-test/int2lm_runjob.cfg

@@ -4,7 +4,7 @@
 #SBATCH --time={walltime}
 #SBATCH --nodes={nodes}
 #SBATCH --ntasks-per-core=1
-#SBATCH --ntasks-per-node={ntasks_per_node}
+#SBATCH --ntasks-per-node={cfg.ntasks_per_node}
 #SBATCH --cpus-per-task=1
 #SBATCH --partition={compute_queue}
 #SBATCH --constraint={cfg.constraint}

jobs/cosmo.py

@@ -66,6 +66,9 @@ def main(starttime, hstart, hstop, cfg, model_cfg):
     tools.create_dir(cfg.cosmo_work, "cosmo_work")
     tools.create_dir(cfg.cosmo_output, "cosmo_output")
 
+    # Total number of processes
+    np_tot = int(cfg.cosmo['np_x'] * cfg.cosmo['np_y'] / cfg.ntasks_per_node) + cfg.cosmo['np_io']
+
     # If an laf* file is used for initialization,
     # copy this to to 'cosmo/input/initial/' or merge with fieldextra
     if hasattr(cfg, 'laf_startfile'):
@@ -142,6 +145,7 @@ def main(starttime, hstart, hstop, cfg, model_cfg):
         tools.create_dir(cfg.cosmo_restart_out, "cosmo_restart_out")
 
     # Copy cosmo executable
+    cfg.cosmo['execname'] = cfg.model.lower()
     tools.copy_file(cfg.cosmo['binary_file'],
                     os.path.join(cfg.cosmo_work, cfg.cosmo['execname']))
 
@@ -162,25 +166,31 @@ def main(starttime, hstart, hstop, cfg, model_cfg):
             namelist_names += ['OAE']
 
     for section in namelist_names:
-        with open(cfg.cosmo['namelist_prefix'] + section + ".cfg") as input_file:
-            to_write = input_file.read()
+        namelist_file = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename,
+                        cfg.cosmo['namelist_prefix'] + section + ".cfg")
+        with open(namelist_file) as input_file:
+            cosmo_namelist = input_file.read()
 
         output_file = os.path.join(cfg.cosmo_work, "INPUT_" + section)
         with open(output_file, "w") as outf:
             if hasattr(cfg, 'spinup'):
                 # no built-in restarts
-                to_write = to_write.format(cfg=cfg,
+                cosmo_namelist = cosmo_namelist.format(cfg=cfg,
+                                           **cfg.cosmo,
+                                           **cfg.oem,
                                            restart_start=12,
                                            restart_stop=0,
                                            restart_step=12)
             else:
                 # built-in restarts
-                to_write = to_write.format(cfg=cfg,
+                cosmo_namelist = cosmo_namelist.format(cfg=cfg,
+                                           **cfg.cosmo,
+                                           **cfg.oem,
                                            restart_start=cfg.hstart +
                                            cfg.restart_step,
                                            restart_stop=cfg.hstop,
                                            restart_step=cfg.restart_step)
-            outf.write(to_write)
+            outf.write(cosmo_namelist)
 
     # Append INPUT_GHG namelist with tracer definitions from csv file
     if os.path.isfile(tracer_csvfile):
@@ -190,13 +200,17 @@ def main(starttime, hstart, hstop, cfg, model_cfg):
             write_cosmo_input_ghg.main(tracer_csvfile, input_ghg_filename, cfg)
 
     # Write run script (run.job)
-    with open(cfg.cosmo['runjob_filename']) as input_file:
-        to_write = input_file.read()
+    runscript_file = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename,
+                     cfg.cosmo['runjob_filename'])
+    with open(runscript_file) as input_file:
+        cosmo_runscript = input_file.read()
 
     output_file = os.path.join(cfg.cosmo_work, "run.job")
     with open(output_file, "w") as outf:
         outf.write(
-            to_write.format(cfg=cfg,
+            cosmo_runscript.format(cfg=cfg,
+                            **cfg.cosmo,
+                            np_tot=np_tot,
                             logfile=logfile,
                             logfile_finish=logfile_finish))
 

run_chain.py

@@ -90,8 +90,8 @@ def __init__(self, casename):
         # Specific settings based on the node type ('gpu' or 'mc')
         self.set_node_info()
 
-        # Check jobs and apply settings
-        self.set_job_variables()
+        # Set some formatted time variables
+        self.set_attributes_time()
 
     def load_config_file(self, casename):
         """
@@ -176,14 +176,6 @@ def set_email(self):
 
         return self
 
-    def set_job_variables(self):
-        self.set_attributes_time()
-
-        if hasattr(self, 'cosmo'):
-            self.set_attributes_cosmo()
-
-        return self
-
     def set_attributes_time(self):
         # ini date and forecast time (ignore meteo times)
         self.inidate_yyyymmddhh = self.startdate.strftime('%Y%m%d%H')
@@ -192,11 +184,6 @@ def set_attributes_time(self):
 
         return self
 
-    def set_attributes_cosmo(self):
-        self.cosmo['execname'] = self.model.lower()
-
-        return self
-
     def print_config(self):
         # Print the configuration
         # max_col_width = max(len(key) for key in vars(self)) + 1