diff --git a/python/Makefile b/python/Makefile index b43e1c5e53..9645242111 100644 --- a/python/Makefile +++ b/python/Makefile @@ -28,6 +28,10 @@ PYLINT_SRC = \ # ../cime_config/buildnml all: test black lint +# ---------------------------------------------------------------- +# See the stest target about this issue + @echo "Run './run_ctsm_py_tests --sys' by hand afterwards" +# ---------------------------------------------------------------- @echo @echo @echo "Successfully ran all standard tests" @@ -40,7 +44,14 @@ utest: FORCE .PHONY: stest stest: FORCE - $(PYTHON) ./run_ctsm_py_tests $(TEST_ARGS) --sys +# ---------------------------------------------------------------- +# EBK 2024-03-19: Comment out running here because of this issue: +# https://github.com/ESCOMP/CTSM/pull/2363#issuecomment-1967884908 +#$(PYTHON) ./run_ctsm_py_tests $(TEST_ARGS) --sys +# Instead run by hand which seems to be working for now... +# ---------------------------------------------------------------- + @echo "System tests currently don't run under Make so..." + @echo "Run './run_ctsm_py_tests --sys' by hand afterwards" .PHONY: lint lint: FORCE diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index ca77faa32c..4af8e66fdd 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -107,377 +107,6 @@ def modify_user_nl(self, case_root, run_type, rundir, site_lines=None): # TODO: include neon-specific user namelist lines, using this as just an example currently if site_lines is None: site_lines = [ -||||||| dac9bad4a - print("using this version:", version) - - if experiment is not None: - self.name = self.name + "." + experiment - case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) - - rundir = None - if os.path.isdir(case_root): - if overwrite: - print("---- removing the existing case -------") - shutil.rmtree(case_root) - elif rerun: - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - # pylint: disable=undefined-variable - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - # pylint: enable=undefined-variable - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print("Case {} appears to be complete, not rerunning.".format(case_root)) - elif not setup_only: - print("Resubmitting case {}".format(case_root)) - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - return - else: - logger.warning("Case already exists in %s, not overwritting", case_root) - return - - if run_type == "postad": - adcase_root = case_root.replace(".postad", ".ad") - if not os.path.isdir(adcase_root): - logger.warning("postad requested but no ad case found in %s", adcase_root) - return - - if not os.path.isdir(case_root): - # read_only = False should not be required here - with Case(base_case_root, read_only=False) as basecase: - print("---- cloning the base case in {}".format(case_root)) - # - # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although - # it causes some of the user_nl_* files to have duplicated inputs. It also ensures - # that the shell_commands file is copied, as well as taking care of the DATM inputs. - # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 - # - basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - - with Case(case_root, read_only=False) as case: - if run_type != "transient": - # in order to avoid the complication of leap years, - # we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") - case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) - if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") - - if run_type == "ad": - case.set_value("CLM_FORCE_COLDSTART", "on") - case.set_value("CLM_ACCELERATED_SPINUP", "on") - case.set_value("RUN_REFDATE", "0018-01-01") - case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("RESUBMIT", 1) - case.set_value("STOP_N", run_length) - - else: - case.set_value("CLM_FORCE_COLDSTART", "off") - case.set_value("CLM_ACCELERATED_SPINUP", "off") - case.set_value("RUN_TYPE", "hybrid") - - if run_type == "postad": - self.set_ref_case(case) - case.set_value("STOP_N", run_length) - - # For transient cases STOP will be set in the user_mod_directory - if run_type == "transient": - if self.finidat: - case.set_value("RUN_TYPE", "startup") - else: - if not self.set_ref_case(case): - return - case.set_value("CALENDAR", "GREGORIAN") - case.set_value("RESUBMIT", 0) - case.set_value("STOP_OPTION", "nmonths") - - if not rundir: - rundir = case.get_value("RUNDIR") - - self.modify_user_nl(case_root, run_type, rundir) - - case.create_namelists() - # explicitly run check_input_data - case.check_all_input_data() - if not setup_only: - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - - def modify_user_nl(self, case_root, run_type, rundir): - """ - Modify user namelist. If transient, include finidat in user_nl; - Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. - """ - user_nl_fname = os.path.join(case_root, "user_nl_clm") - user_nl_lines = None - if run_type == "transient": - if self.finidat: - user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) - ] - else: - user_nl_lines = [ - "hist_fincl2 = ''", - "hist_mfilt = 20", - "hist_nhtfrq = -8760", - "hist_empty_htapes = .true.", -======= - print("using this version:", version) - - if experiment is not None: - self.name = self.name + "." + experiment - case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) - - rundir = None - if os.path.isdir(case_root): - if overwrite: - print("---- removing the existing case -------") - if os.getcwd() == case_root: - abort("Trying to remove the directory tree that we are in") - shutil.rmtree(case_root) - elif rerun: - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - # pylint: disable=undefined-variable - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - # pylint: enable=undefined-variable - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print("Case {} appears to be complete, not rerunning.".format(case_root)) - elif not setup_only: - print("Resubmitting case {}".format(case_root)) - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - return - else: - logger.warning("Case already exists in %s, not overwritting", case_root) - return - - if run_type == "postad": - adcase_root = case_root.replace(".postad", ".ad") - if not os.path.isdir(adcase_root): - logger.warning("postad requested but no ad case found in %s", adcase_root) - return - - if not os.path.isdir(case_root): - # read_only = False should not be required here - with Case(base_case_root, read_only=False) as basecase: - print("---- cloning the base case in {}".format(case_root)) - # - # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although - # it causes some of the user_nl_* files to have duplicated inputs. It also ensures - # that the shell_commands file is copied, as well as taking care of the DATM inputs. - # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 - # - basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - - with Case(case_root, read_only=False) as case: - if run_type != "transient": - # in order to avoid the complication of leap years, - # we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") - case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) - if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") - - if run_type == "ad": - case.set_value("CLM_FORCE_COLDSTART", "on") - case.set_value("CLM_ACCELERATED_SPINUP", "on") - case.set_value("RUN_REFDATE", "0018-01-01") - case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("RESUBMIT", 1) - case.set_value("STOP_N", run_length) - - else: - case.set_value("CLM_FORCE_COLDSTART", "off") - case.set_value("CLM_ACCELERATED_SPINUP", "off") - case.set_value("RUN_TYPE", "hybrid") - - if run_type == "postad": - self.set_ref_case(case) - case.set_value("STOP_N", run_length) - - # For transient cases STOP will be set in the user_mod_directory - if run_type == "transient": - if self.finidat: - case.set_value("RUN_TYPE", "startup") - else: - if not self.set_ref_case(case): - return - case.set_value("CALENDAR", "GREGORIAN") - case.set_value("RESUBMIT", 0) - case.set_value("STOP_OPTION", "nmonths") - - if not rundir: - rundir = case.get_value("RUNDIR") - - self.modify_user_nl(case_root, run_type, rundir) - - case.create_namelists() - # explicitly run check_input_data - case.check_all_input_data() - if not setup_only: - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - - def modify_user_nl(self, case_root, run_type, rundir): - """ - Modify user namelist. If transient, include finidat in user_nl; - Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. - """ - user_nl_fname = os.path.join(case_root, "user_nl_clm") - user_nl_lines = None - if run_type == "transient": - if self.finidat: - user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) - ] - else: - user_nl_lines = [ - "hist_fincl2 = ''", - "hist_mfilt = 20", - "hist_nhtfrq = -8760", - "hist_empty_htapes = .true.", ->>>>>>> escomp/b4b-dev """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO',""" ] diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index d92df0fec1..1679df83e9 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -21,6 +21,7 @@ # pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root +from ctsm.utils import abort from CIME import build from CIME.case import Case