From fdeb2b505d214b226f8529127072a83ee38af205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 19 Sep 2023 15:03:35 +0200 Subject: [PATCH] Remove `variants` key and use `features` instead (#30) * Remove variants from models.yaml * Remove spinup variant and use the spinup variable instead * GitHub Action: Apply Pep8-formatting * Remove variant check * Install specific icontools version --------- Co-authored-by: github-actions --- cases/cosmo-ghg-11km-test/config.py | 1 - config/models.yaml | 13 ++---- jenkins/scripts/setup-spack.sh | 2 +- jobs/cosmo.py | 12 ++--- jobs/post_int2lm.py | 2 +- jobs/tools/write_cosmo_input_ghg.py | 2 +- run_chain.py | 69 +++++++---------------------- 7 files changed, 29 insertions(+), 72 deletions(-) diff --git a/cases/cosmo-ghg-11km-test/config.py b/cases/cosmo-ghg-11km-test/config.py index 0e59d148..6e08ca7e 100644 --- a/cases/cosmo-ghg-11km-test/config.py +++ b/cases/cosmo-ghg-11km-test/config.py @@ -18,7 +18,6 @@ model = 'cosmo-ghg' restart_step = 12 # hours -variant = 'spinup' spinup = 6 if constraint == 'gpu': diff --git a/config/models.yaml b/config/models.yaml index a01c0854..a19f188c 100644 --- a/config/models.yaml +++ b/config/models.yaml @@ -1,31 +1,24 @@ models: cosmo: jobs: ['prepare_data', 'int2lm', 'cosmo', 'post_cosmo'] - variants: ['spinup'] - features: ['restart'] + features: ['restart', 'spinup'] cosmo-ghg: jobs: ['prepare_data', 'emissions', 'biofluxes', 'oem', 'online_vprm', 'int2lm', 'post_int2lm', 'cosmo', 'post_cosmo'] - variants: ['spinup'] - features: ['restart', 'tracers'] + features: ['restart', 'tracers', 'spinup'] cosmo-art: jobs: ['prepare_data', 'emissions', 'obs_nudging', 'photo_rate', 'int2lm', 'cosmo', 'post_cosmo'] - variants: ['spinup'] - features: ['nesting'] + features: ['nesting', 'spinup'] icon: jobs: ['prepare_data', 'icon'] - variants: [] features: ['restart'] icon-art: jobs: ['prepare_data', 'icon'] - variants: [] features: ['restart'] icon-art-global: jobs: ['prepare_data', 'icon'] - variants: [] features: ['restart'] icon-art-oem: jobs: ['prepare_data', 'icon'] - variants: [] features: ['restart'] diff --git a/jenkins/scripts/setup-spack.sh b/jenkins/scripts/setup-spack.sh index 9f8efef2..93945c0b 100755 --- a/jenkins/scripts/setup-spack.sh +++ b/jenkins/scripts/setup-spack.sh @@ -20,5 +20,5 @@ git clone --depth 1 --recurse-submodules --shallow-submodules -b ${TAG} ${GIT_RE . spack-c2sm/setup-env.sh popd -spack install icontools +spack install icontools@c2sm-master%gcc diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 75780dc4..462b44ed 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -142,9 +142,10 @@ def main(starttime, hstart, hstop, cfg, model_cfg): "Start time %s must not be smaller than in laf_starttime %s." % (str(starttime), str(startfiletime))) - # No restarts for COSMO-ART and for simulations with spinup - if 'restart' in model_cfg['models'][cfg.model]['features'] and \ - cfg.variant != 'spinup': + # Create restart directory if feature is present and + # if there is no spinup + if 'restart' in model_cfg['models'][cfg.model]['features'] and not \ + hasattr(cfg, 'spinup'): tools.create_dir(cfg.cosmo_restart_out, "cosmo_restart_out") # Copy cosmo executable @@ -174,13 +175,14 @@ def main(starttime, hstart, hstop, cfg, model_cfg): output_file = os.path.join(cfg.cosmo_work, "INPUT_" + section) with open(output_file, "w") as outf: - if cfg.variant == 'spinup': - # no restarts + if hasattr(cfg, 'spinup'): + # no built-in restarts to_write = to_write.format(cfg=cfg, restart_start=12, restart_stop=0, restart_step=12) else: + # built-in restarts to_write = to_write.format(cfg=cfg, restart_start=cfg.hstart + cfg.restart_step, diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 57cffbf6..61f30c15 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -90,7 +90,7 @@ def main(start_time, hstart, hstop, cfg, model_cfg): logging.info("OK") # Meteo spinup simulation with tracer recycling - if cfg.variant == 'spinup' and \ + if hasattr(cfg, 'spinup') and \ hasattr(cfg, 'post_int2lm_species_spinup') and not cfg.first_one: var_list = cfg.post_int2lm_species_spinup logging.info( diff --git a/jobs/tools/write_cosmo_input_ghg.py b/jobs/tools/write_cosmo_input_ghg.py index 59df37fb..e0a11ef3 100644 --- a/jobs/tools/write_cosmo_input_ghg.py +++ b/jobs/tools/write_cosmo_input_ghg.py @@ -104,7 +104,7 @@ def main(csv_filename, namelist_filename, cfg=None): with open(namelist_filename, 'a') as nml_file: for group in reader: - if cfg.variant == 'spinup' and not cfg.first_one: + if hasattr(cfg, 'spinup') and not cfg.first_one: nml_file.write(group2text(group, recycling=True)) else: nml_file.write(group2text(group)) diff --git a/run_chain.py b/run_chain.py index 5582ae75..e09f3ba7 100755 --- a/run_chain.py +++ b/run_chain.py @@ -144,42 +144,6 @@ def load_config_file(casename, cfg): return cfg -def check_model_set_variant(model_cfg, cfg): - """Checks the model and sets its variant. - - Check if a model was provided in the config-object. If no model is - provided, set the model to cosmo in the config-object. - - Check if a variant was provided in the config-object. Variants - provide a way to customize the behaviour of the processing chain - for different types of simulations. - - Raise a RuntimeError if an unsupported model or variant is given in cfg. - You can add models and variants in the config/models.yaml file. - - Parameters - ---------- - cfg : config-object - """ - if hasattr(cfg, 'model'): - model_str = getattr(cfg, 'model') - else: - raise RuntimeError("Variable 'model' not set in config.") - - models = model_cfg['models'] - if cfg.model not in models: - raise ValueError("Invalid model: {}".format(model_str)) - - if hasattr(cfg, 'variant'): - variants = models[cfg.model]['variants'] - if cfg.variant not in variants: - raise ValueError(f"Invalid variant for {cfg.model}: {cfg.variant}") - else: - setattr(cfg, 'variant', None) - - return cfg - - def run_chain(work_root, model_cfg, cfg, start_time, hstart, hstop, job_names, force): """Run chain ignoring already finished jobs. @@ -242,7 +206,7 @@ def run_chain(work_root, model_cfg, cfg, start_time, hstart, hstop, job_names, inidate_int2lm_yyyymmddhh = (start_time + timedelta(hours=hstart)).strftime('%Y%m%d%H') - if cfg.variant == 'spinup': + if hasattr(cfg, 'spinup'): if cfg.first_one: # first run in spinup chain_root_last_run = '' else: # consecutive runs in spinup @@ -308,7 +272,7 @@ def run_chain(work_root, model_cfg, cfg, start_time, hstart, hstop, job_names, raise FileNotFoundError(f"File not found: {tracer_csvfile}") # tracer_start namelist paramter for spinup simulation - if cfg.variant == 'spinup': + if hasattr(cfg, 'spinup'): if cfg.first_one: setattr(cfg, 'tracer_start', 0) else: @@ -331,7 +295,7 @@ def run_chain(work_root, model_cfg, cfg, start_time, hstart, hstop, job_names, "gpu or mc") # Spinup - if cfg.variant == 'spinup': + if hasattr(cfg, 'spinup'): setattr(cfg, 'last_cosmo_output', os.path.join(chain_root_last_run, 'cosmo', 'output')) # No restart for spinup simulations (= default values for no restart) @@ -639,24 +603,15 @@ def load_model_config_yaml(yamlfile): model_cfg = load_model_config_yaml('config/models.yaml') cfg = load_config_file(casename=casename, cfg=cfg) start_time = datetime.strptime(args.startdate, '%Y-%m-%d') - cfg = check_model_set_variant(model_cfg, cfg) if args.job_list is None: args.job_list = model_cfg['models'][cfg.model]['jobs'] print(f"Starting chain for case {casename} and model {cfg.model}") + # check for restart compatibility and spinup if 'restart' in model_cfg['models'][cfg.model]['features']: - if cfg.variant is None: - restart_runs(work_root=cfg.work_root, - model_cfg=model_cfg, - cfg=cfg, - start=start_time, - hstart=args.hstart, - hstop=args.hstop, - job_names=args.job_list, - force=args.force) - elif cfg.variant == 'spinup': - print(f"Model variant is {cfg.variant}") + if hasattr(cfg, 'spinup'): + print("This is a spinup simulation.") restart_runs_spinup(work_root=cfg.work_root, model_cfg=model_cfg, cfg=cfg, @@ -666,9 +621,17 @@ def load_model_config_yaml(yamlfile): job_names=args.job_list, force=args.force) else: - raise RuntimeError(f"Unknown variant: {cfg.variant}") + print("Built-in model restart is used.") + restart_runs(work_root=cfg.work_root, + model_cfg=model_cfg, + cfg=cfg, + start=start_time, + hstart=args.hstart, + hstop=args.hstop, + job_names=args.job_list, + force=args.force) else: - # cosmoart can't do restarts + print("No restart is used.") run_chain(work_root=cfg.work_root, cfg=cfg, start_time=start_time,