Merge pull request ESCOMP#2334 from slevis-lmwg/bfb_merge_tag_2315_et_al

Bfb merge tag dev166

.git-blame-ignore-revs

@@ -21,6 +21,7 @@ e4d38681df23ccca0ae29581a45f8362574e0630
 025d5e7c2e80263717fb029101d65cbbf261c3c4
 a9d96219902cf609636886c7073a84407f450d9a
 d866510188d26d51bcd6d37239283db690af7e82
+0dcd0a3c1abcaffe5529f8d79a6bc34734b195c7
 # Ran SystemTests and python/ctsm through black python formatter
 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c
 8056ae649c1b37f5e10aaaac79005d6e3a8b2380

.github/workflows/assign-to-project.yml

@@ -0,0 +1,23 @@
+name: Auto Assign to Project(s)
+
+on:
+  issues:
+    types: [opened, labeled]
+  pull_request:
+    types: [opened, labeled]
+  issue_comment:
+    types: [created]
+
+jobs:
+  assign_high_priority:
+    runs-on: ubuntu-latest
+    name: Assign to High Priority project
+    steps:
+    - name: Assign issues and pull requests with `priority: high` label to project 25
+      uses: srggrs/[email protected]
+      if: |
+        contains(github.event.issue.labels.*.name, 'priority: high') ||
+        contains(github.event.pull_request.labels.*.name, 'priority: high')
+      with:
+        project: 'https://github.com/ESCOMP/CTSM/projects/25'
+        column_name: 'Needs triage'

cime_config/testdefs/testlist_clm.xml

@@ -3433,6 +3433,7 @@
     <machines>
       <machine name="cheyenne" compiler="intel" category="aux_clm"/>
       <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="izumi" compiler="intel" category="aux_clm"/>
     </machines>
     <options>
       <option name="wallclock">00:30:00</option>

doc/ChangeLog

@@ -1,4 +1,86 @@
 ===============================================================
+Tag name: ctsm5.1.dev166
+Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310), tking (Teagan King), samrabin (Sam Rabin)
+Date: Wed 24 Jan 2024 05:39:41 PM MST
+One-line Summary: BFB merge tag
+
+Purpose and description of changes
+----------------------------------
+
+ #2315 @TeaganKing Refactoring run_neon for PLUMBER2 part1
+ #2213 @samsrabin Automatically assign high priority items to project 25
+ #2330 @samsrabin Add Izumi version of the aux_clm unit testing
+ #2326 @samsrabin run_sys_tests: Check Python environment for FatesColdTwoStream tests
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm5_1
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed
+----------
+
+CTSM issues fixed (include CTSM Issue #):
+ Fixes #2315
+ Fixes #2213
+ Fixes #2330
+ Fixes #2326
+
+Known bugs introduced in this tag (include issue #):
+ - New feature coming in with #2213 where user will receive email from
+ github when pushing to their remote:
+ "Run failed: .github/workflows/assign-to-project.yml"
+ - New feature that also affects older tags: The izumi FatesColdTwoStream
+ test submitted from ./run_sys_tests will fail at CREATE_NEWCASE unless users
+ introduce "module load lang/python/3.7.0" in their .bash_profile.
+ Longterm solution discussed in #2335. The test also works when submitted
+ manually with ./create_test.
+
+Notes of particular relevance for developers:
+---------------------------------------------
+Changes to tests or testing:
+ #2315 New unit tests for arg_parse and NeonSite
+ #2330 New test in aux_clm that does unit testing on izumi because unit
+       testing does not work on derecho, yet
+
+Testing summary:
+----------------
+
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    derecho - OK, pylint gives long list of warnings (expected)
+
+  regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing):
+
+    derecho ----- OK
+    izumi ------- OK
+
+
+Answer changes
+--------------
+Changes answers relative to baseline: No
+
+Other details
+-------------
+Pull Requests that document the changes (include PR ids):
+ https://github.com/ESCOMP/ctsm/pull/2334
+
+===============================================================
+===============================================================
 Tag name: ctsm5.1.dev165
 Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310), oleson (Keith Oleson), samrabin (Sam Rabin)
 Date: Fri 19 Jan 2024 06:40:36 PM MST

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+    ctsm5.1.dev166 multiple 01/24/2024 BFB merge tag
     ctsm5.1.dev165   slevis 01/19/2024 Turn Meier2022, tillage, residue removal on for ctsm5.1, fix #2212
     ctsm5.1.dev164   rgknox 01/17/2024 Compatibility and tests for FATES 2-Stream
     ctsm5.1.dev163 samrabin 01/10/2024 Add tillage and residue removal

python/ctsm/run_sys_tests.py

@@ -249,7 +249,7 @@ def run_sys_tests(
         else:
             raise RuntimeError("None of suite_name, testfile or testlist were provided")
         if not running_ctsm_py_tests:
-            _try_systemtests(testname_list)
+            _check_py_env(testname_list)
         _run_create_test(
             cime_path=cime_path,
             test_args=test_args,
@@ -708,20 +708,55 @@ def _run_test_suite(
         )
 
 
-def _try_systemtests(testname_list):
+def _get_testmod_list(test_attributes, unique=False):
+    # Isolate testmods, producing a list like
+    # ["clm-test1mod1", "clm-test2mod1", "clm-test2mod2", ...]
+    # Handles test attributes passed in from run_sys_tests calls using -t, -f, or -s
+
+    testmods = []
+    for test_attribute in test_attributes:
+        for dot_split in test_attribute.split("."):
+            slash_replaced = dot_split.replace("/", "-")
+            for ddash_split in slash_replaced.split("--"):
+                if "clm-" in ddash_split and (ddash_split not in testmods or not unique):
+                    testmods.append(ddash_split)
+
+    return testmods
+
+
+def _check_py_env(test_attributes):
     err_msg = " can't be loaded. Do you need to activate the ctsm_pylib conda environment?"
     # Suppress pylint import-outside-toplevel warning because (a) we only want to import
     # this when certain tests are requested, and (b) the import needs to be in a try-except
     # block to produce a nice error message.
     # pylint: disable=import-outside-toplevel disable
     # Suppress pylint unused-import warning because the import itself IS the use.
     # pylint: disable=unused-import disable
-    if any("FSURDATMODIFYCTSM" in t for t in testname_list):
+    # Suppress pylint import-error warning because the whole point here is to check
+    # whether import is possible.
+    # pylint: disable=import-error disable
+
+    # Check requirements for FSURDATMODIFYCTSM, if needed
+    if any("FSURDATMODIFYCTSM" in t for t in test_attributes):
         try:
             import ctsm.modify_input_files.modify_fsurdat
         except ModuleNotFoundError as err:
             raise ModuleNotFoundError("modify_fsurdat" + err_msg) from err
 
+    # Check that list for any testmods that use modify_fates_paramfile.py
+    testmods_to_check = ["clm-FatesColdTwoStream", "clm-FatesColdTwoStreamNoCompFixedBioGeo"]
+    testmods = _get_testmod_list(test_attributes)
+    if any(t in testmods_to_check for t in testmods):
+        # This bit is needed because it's outside the top-level python/ directory.
+        fates_dir = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "src", "fates"
+        )
+        sys.path.insert(1, fates_dir)
+        try:
+            import tools.modify_fates_paramfile
+        except ModuleNotFoundError as err:
+            raise ModuleNotFoundError("modify_fates_paramfile" + err_msg) from err
+
 
 def _get_compilers_for_suite(suite_name, machine_name, running_ctsm_py_tests):
     test_data = get_tests_from_xml(xml_machine=machine_name, xml_category=suite_name)
@@ -730,7 +765,8 @@ def _get_compilers_for_suite(suite_name, machine_name, running_ctsm_py_tests):
             "No tests found for suite {} on machine {}".format(suite_name, machine_name)
         )
     if not running_ctsm_py_tests:
-        _try_systemtests([t["testname"] for t in test_data])
+        _check_py_env([t["testname"] for t in test_data])
+        _check_py_env([t["testmods"] for t in test_data if "testmods" in t.keys()])
     compilers = sorted({one_test["compiler"] for one_test in test_data})
     logger.info("Running with compilers: %s", compilers)
     return compilers