diff --git a/cases/cosmo-ghg-11km-test/config.py b/cases/cosmo-ghg-11km-test/config.py index 6e08ca7e..679bd7f9 100644 --- a/cases/cosmo-ghg-11km-test/config.py +++ b/cases/cosmo-ghg-11km-test/config.py @@ -129,7 +129,6 @@ # Domain decomposition int2lm_nodes = 2 -int2lm_ntasks_per_node = 12 int2lm_np_x = 8 int2lm_np_y = 3 int2lm_np_tot = int2lm_np_x * int2lm_np_y diff --git a/cases/cosmo-ghg-11km-test/config.yaml b/cases/cosmo-ghg-11km-test/config.yaml index dd573634..c3387e02 100644 --- a/cases/cosmo-ghg-11km-test/config.yaml +++ b/cases/cosmo-ghg-11km-test/config.yaml @@ -2,6 +2,7 @@ model: cosmo-ghg constraint: gpu +ntasks_per_node: 12 restart_step: 12 variant: spinup spinup: 6 @@ -53,7 +54,6 @@ int2lm: compute_queue: normal walltime: 01:00:00 nodes: 2 - ntasks_per_node: 12 np_x: 8 np_y: 3 @@ -73,6 +73,7 @@ cosmo: binary_file: ./src/cosmo-ghg/cosmo/ACC/cosmo_gpu namelist_prefix: cosmo_INPUT_ runjob_filename: cosmo_runjob.cfg + compute_queue: normal walltime: "00:30:00" np_x: 6 np_y: 4 diff --git a/cases/cosmo-ghg-11km-test/cosmo_INPUT_GHG.cfg b/cases/cosmo-ghg-11km-test/cosmo_INPUT_GHG.cfg index 505ba793..2739d66e 100644 --- a/cases/cosmo-ghg-11km-test/cosmo_INPUT_GHG.cfg +++ b/cases/cosmo-ghg-11km-test/cosmo_INPUT_GHG.cfg @@ -2,11 +2,11 @@ in_tracers = {cfg.in_tracers}, tracer_start = {cfg.tracer_start}, iemiss_interp = 1, - gridded_emissions_nc = '../input/oem/{cfg.oem_gridded_emissions_nc}', - vertical_profile_nc = '../input/oem/{cfg.oem_vertical_profiles_nc}', - hour_of_day_nc = '../input/oem/{cfg.oem_hourofday_nc}', - day_of_week_nc = '../input/oem/{cfg.oem_dayofweek_nc}', - month_of_year_nc = '../input/oem/{cfg.oem_monthofyear_nc}', + gridded_emissions_nc = '../input/oem/{gridded_emissions_nc}', + vertical_profile_nc = '../input/oem/{vertical_profiles_nc}', + hour_of_day_nc = '../input/oem/{hourofday_nc}', + day_of_week_nc = '../input/oem/{dayofweek_nc}', + month_of_year_nc = '../input/oem/{monthofyear_nc}', modis_reflectances_nc = '../input/vprm/modis.nc', veg_class_frac_nc = '../input/vprm/vegetation.nc', octe_maps_nc = '../input/octe/maps.nc', diff --git a/cases/cosmo-ghg-11km-test/cosmo_INPUT_ORG.cfg b/cases/cosmo-ghg-11km-test/cosmo_INPUT_ORG.cfg index fd1b96df..2f943a03 100644 --- a/cases/cosmo-ghg-11km-test/cosmo_INPUT_ORG.cfg +++ b/cases/cosmo-ghg-11km-test/cosmo_INPUT_ORG.cfg @@ -15,8 +15,8 @@ hstart = {cfg.hstart}, hstop = {cfg.hstop}, ydate_ini = '{cfg.inidate_yyyymmddhh}', - nprocx = {cfg.cosmo_np_x}, - nprocy = {cfg.cosmo_np_y}, + nprocx = {np_x}, + nprocy = {np_y}, nprocio = 0, nproma = -1, hincmxt = 1.0, @@ -30,7 +30,7 @@ ldfi = .FALSE., ldump_ascii = .FALSE., ncomm_type = 1, ! (OF: we use 1 operationally since 2 was causing trouble with memory consumption, leak in the MPI library) - num_asynio_comm = {cfg.cosmo_np_io}, + num_asynio_comm = {np_io}, num_iope_percomm = 0, idbg_level = 8, nboundlines = 3, diff --git a/cases/cosmo-ghg-11km-test/cosmo_runjob.cfg b/cases/cosmo-ghg-11km-test/cosmo_runjob.cfg index 3b0d7118..1efeb7c3 100644 --- a/cases/cosmo-ghg-11km-test/cosmo_runjob.cfg +++ b/cases/cosmo-ghg-11km-test/cosmo_runjob.cfg @@ -1,12 +1,12 @@ #!/bin/bash -l #SBATCH --job-name="cosmo_{cfg.inidate_yyyymmddhh}_{cfg.forecasttime}" #SBATCH --account={cfg.compute_account} -#SBATCH --time={cfg.cosmo_walltime} -#SBATCH --nodes={cfg.cosmo_np_tot} +#SBATCH --time={walltime} +#SBATCH --nodes={np_tot} #SBATCH --ntasks-per-core=1 #SBATCH --ntasks-per-node={cfg.ntasks_per_node} #SBATCH --cpus-per-task=1 -#SBATCH --partition={cfg.compute_queue} +#SBATCH --partition={compute_queue} #SBATCH --constraint={cfg.constraint} #SBATCH --hint=nomultithread #SBATCH --output={logfile} @@ -34,7 +34,7 @@ echo "============== StartTime: `date +%s` s" echo "============== StartTime: `date`" echo "=====================================================" -srun -u ./{cfg.execname} >> {logfile} 2>&1 +srun -u ./{execname} >> {logfile} 2>&1 pid=$? echo "=====================================================" diff --git a/cases/cosmo-ghg-11km-test/int2lm_runjob.cfg b/cases/cosmo-ghg-11km-test/int2lm_runjob.cfg index 1da5db36..e702bc92 100644 --- a/cases/cosmo-ghg-11km-test/int2lm_runjob.cfg +++ b/cases/cosmo-ghg-11km-test/int2lm_runjob.cfg @@ -4,7 +4,7 @@ #SBATCH --time={walltime} #SBATCH --nodes={nodes} #SBATCH --ntasks-per-core=1 -#SBATCH --ntasks-per-node={ntasks_per_node} +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} #SBATCH --cpus-per-task=1 #SBATCH --partition={compute_queue} #SBATCH --constraint={cfg.constraint} diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 1cb03c42..2b1ffd5b 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -66,6 +66,9 @@ def main(starttime, hstart, hstop, cfg, model_cfg): tools.create_dir(cfg.cosmo_work, "cosmo_work") tools.create_dir(cfg.cosmo_output, "cosmo_output") + # Total number of processes + np_tot = int(cfg.cosmo['np_x'] * cfg.cosmo['np_y'] / cfg.ntasks_per_node) + cfg.cosmo['np_io'] + # If an laf* file is used for initialization, # copy this to to 'cosmo/input/initial/' or merge with fieldextra if hasattr(cfg, 'laf_startfile'): @@ -142,6 +145,7 @@ def main(starttime, hstart, hstop, cfg, model_cfg): tools.create_dir(cfg.cosmo_restart_out, "cosmo_restart_out") # Copy cosmo executable + cfg.cosmo['execname'] = cfg.model.lower() tools.copy_file(cfg.cosmo['binary_file'], os.path.join(cfg.cosmo_work, cfg.cosmo['execname'])) @@ -162,25 +166,31 @@ def main(starttime, hstart, hstop, cfg, model_cfg): namelist_names += ['OAE'] for section in namelist_names: - with open(cfg.cosmo['namelist_prefix'] + section + ".cfg") as input_file: - to_write = input_file.read() + namelist_file = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename, + cfg.cosmo['namelist_prefix'] + section + ".cfg") + with open(namelist_file) as input_file: + cosmo_namelist = input_file.read() output_file = os.path.join(cfg.cosmo_work, "INPUT_" + section) with open(output_file, "w") as outf: if hasattr(cfg, 'spinup'): # no built-in restarts - to_write = to_write.format(cfg=cfg, + cosmo_namelist = cosmo_namelist.format(cfg=cfg, + **cfg.cosmo, + **cfg.oem, restart_start=12, restart_stop=0, restart_step=12) else: # built-in restarts - to_write = to_write.format(cfg=cfg, + cosmo_namelist = cosmo_namelist.format(cfg=cfg, + **cfg.cosmo, + **cfg.oem, restart_start=cfg.hstart + cfg.restart_step, restart_stop=cfg.hstop, restart_step=cfg.restart_step) - outf.write(to_write) + outf.write(cosmo_namelist) # Append INPUT_GHG namelist with tracer definitions from csv file if os.path.isfile(tracer_csvfile): @@ -190,13 +200,17 @@ def main(starttime, hstart, hstop, cfg, model_cfg): write_cosmo_input_ghg.main(tracer_csvfile, input_ghg_filename, cfg) # Write run script (run.job) - with open(cfg.cosmo['runjob_filename']) as input_file: - to_write = input_file.read() + runscript_file = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename, + cfg.cosmo['runjob_filename']) + with open(runscript_file) as input_file: + cosmo_runscript = input_file.read() output_file = os.path.join(cfg.cosmo_work, "run.job") with open(output_file, "w") as outf: outf.write( - to_write.format(cfg=cfg, + cosmo_runscript.format(cfg=cfg, + **cfg.cosmo, + np_tot=np_tot, logfile=logfile, logfile_finish=logfile_finish)) diff --git a/run_chain.py b/run_chain.py index c886520c..8167bde4 100755 --- a/run_chain.py +++ b/run_chain.py @@ -90,8 +90,8 @@ def __init__(self, casename): # Specific settings based on the node type ('gpu' or 'mc') self.set_node_info() - # Check jobs and apply settings - self.set_job_variables() + # Set some formatted time variables + self.set_attributes_time() def load_config_file(self, casename): """ @@ -176,14 +176,6 @@ def set_email(self): return self - def set_job_variables(self): - self.set_attributes_time() - - if hasattr(self, 'cosmo'): - self.set_attributes_cosmo() - - return self - def set_attributes_time(self): # ini date and forecast time (ignore meteo times) self.inidate_yyyymmddhh = self.startdate.strftime('%Y%m%d%H') @@ -192,11 +184,6 @@ def set_attributes_time(self): return self - def set_attributes_cosmo(self): - self.cosmo['execname'] = self.model.lower() - - return self - def print_config(self): # Print the configuration # max_col_width = max(len(key) for key in vars(self)) + 1