Update ufs-bundle to build with head of develop of ufs-weather-model …

…for ATM configuration (#43) This PR, in combination with a PRfor fv3-jedi, allows the ufs-bundle to be built using the head of develop for the ufs-weather-model. Additionally, new regression tests have been added to fv3-jedi that will update the run directories needed for the UFS model and to run a sample Ensemble Forecast of the UFS.
JCSDA · Feb 2, 2024 · 11410cc · 11410cc
1 parent f19b55d
commit 11410cc
Show file tree

Hide file tree

Showing 4 changed files with 334 additions and 55 deletions.
diff --git a/.github/workflows/run_ec2_pcluster.yaml b/.github/workflows/run_ec2_pcluster.yaml
@@ -90,6 +90,7 @@ jobs:
           module load stack-python/3.10.8
 
           module load jedi-ufs-env/1.0.0
+          module load fms/2023.02.01
 
           module li
 
@@ -128,20 +129,17 @@ jobs:
 
           # UFS_APP=ATM
           cd ${JEDI_ENV}
+          rm -rf build-*
           mkdir -p build-atm
           cd build-atm
           cmake -DUFS_APP=ATM ../ufs-bundle
           make update
           make -j8
 
           # Runs mandatory ctests on the head node
-          ctest -R get_
-          ctest -R fv3jedi_setup_ufs_rundir_warmstart
-          ctest -R fv3jedi_setup_ufs_rundir_coldstart
-          ctest -R fv3jedi_test_tier1_model_ufs_warmstart
-          #ctest -R fv3jedi_test_tier1_model_ufs_coldstart
-          ctest -R fv3jedi_test_tier1_forecast_ufs_warmstart
-          #ctest -R fv3jedi_test_tier1_forecast_ufs_coldstart
+          rm -rf fv3-jedi/test/Data/ModelRunDirs/UFS_warmstart/*
+          rm -rf fv3-jedi/test/Data/ModelRunDirs/UFS_warmstart_2/*
+          ctest -R ufs_
 
           # UFS_APP=ATMAERO
           cd ${JEDI_ENV}
@@ -152,21 +150,21 @@ jobs:
           make -j8
 
           # Skip those for now - linker errors because of static/dynamic library mix
-          ## UFS_APP=NG-GODAS
-          #cd ${JEDI_ENV}
-          #mkdir -p build-ng-godas
-          #cd build-ng-godas
-          #cmake -DUFS_APP=NG-GODAS ../ufs-bundle
-          #make update
-          #make -j8
+          UFS_APP=NG-GODAS
+          cd ${JEDI_ENV}
+          mkdir -p build-ng-godas
+          cd build-ng-godas
+          cmake -DUFS_APP=NG-GODAS ../ufs-bundle
+          make update
+          make -j8
           #
-          ## UFS_APP=S2S
-          #cd ${JEDI_ENV}
-          #mkdir -p build-s2s
-          #cd build-s2s
-          #cmake -DUFS_APP=S2S ../ufs-bundle
-          #make update
-          #make -j8
+          # UFS_APP=S2S
+          cd ${JEDI_ENV}
+          mkdir -p build-s2s
+          cd build-s2s
+          cmake -DUFS_APP=S2S ../ufs-bundle
+          make update
+          make -j8
 
       - name: Report Status
         if: always()

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -5,6 +5,7 @@
 # ##########
 
 cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
+include( ExternalProject )
 
 find_package( ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
 
@@ -41,6 +42,7 @@ find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8)
 
 # Core JEDI repositories
 # ----------------------
+#ecbuild_bundle( PROJECT oops  GIT "https://github.com/jcsda-internal/oops.git"  BRANCH feature/ufs-stateset)
 ecbuild_bundle( PROJECT oops  GIT "https://github.com/jcsda-internal/oops.git"  BRANCH develop UPDATE )
 ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE )
 ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE )
@@ -79,8 +81,8 @@ include_directories(${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_INCLUDEDIR})
 include_directories(${DEPEND_LIB_ROOT}/include_r8)
 link_directories(${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR})
 
-include( ExternalProject )
 
+# Set variables for UFS clone and build
 # Needed to get correct OpenMP link libraries on macOS
 set(UFS_CMAKE_EXE_LINKER_FLAGS "-L${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
 find_package(OpenMP REQUIRED)
@@ -96,12 +98,23 @@ else()
   set(UFS_CMAKE_BUILD_TYPE_FLAG "-DREPRO=ON")
 endif()
 
+
+set(UFS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model)
+set(UFS_REPO https://github.com/ufs-community/ufs-weather-model)
+set(UFS_TAG "develop" )
+if(UFS_APP MATCHES "^(NG-GODAS)$")
+   set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ng-godas.patch <SOURCE_DIR> && cd <SOURCE_DIR> && git apply ng-godas.patch)
+else()
+  set(patch_command "")
+endif()
+
 ExternalProject_Add(ufs-weather-model
   PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model
-  GIT_REPOSITORY https://github.com/climbfuji/ufs-weather-model
+  GIT_REPOSITORY ${UFS_REPO}
   GIT_SUBMODULES_RECURSE TRUE
-  GIT_TAG feature/consolidate_dom_update_20221114  # updated from develop on 2023/07/17
-  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model
+  GIT_TAG ${UFS_TAG}
+  PATCH_COMMAND ${patch_command}
+  SOURCE_DIR ${UFS_SOURCE_DIR}
   UPDATE_DISCONNECTED ON
   INSTALL_DIR ${DEPEND_LIB_ROOT}
   # DH* 20230316 turn off OpenMP for now ...
@@ -133,37 +146,35 @@ add_library( stochastic_physics IMPORTED STATIC)
 add_library( fv3atm IMPORTED STATIC)
 add_library( ccppphys IMPORTED STATIC)
 add_library( mom6 IMPORTED STATIC)
-set_target_properties( ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libufs.a)
-set_target_properties( ufs_aerosols PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libUFS_Aerosols.a)
-set_target_properties( aerosols_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libAerosol_GridComp.a)
-set_target_properties( gocart_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libGOCART2G_GridComp.a)
-set_target_properties( ca2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libCA2G_GridComp.a)
-set_target_properties( chem_shared PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libChem_Shared2G.a)
-set_target_properties( gocart_process PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libProcess_Library.a)
-set_target_properties( du2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libDU2G_GridComp.a)
-set_target_properties( ni2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libNI2G_GridComp.a)
-set_target_properties( ss2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libSS2G_GridComp.a)
-set_target_properties( su2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libSU2G_GridComp.a)
-set_target_properties( ga_environment PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libGA_Environment.a)
-set_target_properties( cdeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libcdeps.a)
-set_target_properties( cmeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libcmeps.a)
-set_target_properties(ccpp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libccpp_framework.a)
-set_target_properties(stochastic_physics PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libstochastic_physics.a)
-set_target_properties(fv3atm PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libfv3atm.a)
-set_target_properties(ccppphys PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libccpp_physics.a)
-set_target_properties(mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libmom6.a)
-
-if(UFS_APP MATCHES "^(ATM)$" OR UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$")
-  ecbuild_bundle( PROJECT femps         GIT "https://github.com/jcsda-internal/femps.git"         TAG 1.2.0 )
+set_target_properties( ufs PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libufs.a )
+set_target_properties( ufs_aerosols PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libUFS_Aerosols.a )
+set_target_properties( aerosols_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libAerosol_GridComp.a )
+set_target_properties( gocart_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libGOCART2G_GridComp.a )
+set_target_properties( ca2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libCA2G_GridComp.a )
+set_target_properties( chem_shared PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libChem_Shared2G.a )
+set_target_properties( gocart_process PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libProcess_Library.a )
+set_target_properties( du2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libDU2G_GridComp.a )
+set_target_properties( ni2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libNI2G_GridComp.a )
+set_target_properties( ss2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libSS2G_GridComp.a )
+set_target_properties( su2g_gridcomp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libSU2G_GridComp.a )
+set_target_properties( ga_environment PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libGA_Environment.a )
+set_target_properties( cdeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libcdeps.a )
+set_target_properties( cmeps PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libcmeps.a )
+set_target_properties( ccpp PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libccpp_framework.a )
+set_target_properties( stochastic_physics PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libstochastic_physics.a )
+set_target_properties( fv3atm PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libfv3atm.a )
+set_target_properties( ccppphys PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libccpp_physics.a )
+set_target_properties( mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_LIBDIR}/libmom6.a )
+
+
+
+ecbuild_bundle( PROJECT femps    GIT "https://github.com/jcsda-internal/femps.git"    TAG 1.2.0 )
+if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(ATM)$")
   option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/ufs_dom UPDATE )  # updated from develop Dec 05 2023
-  ecbuild_bundle( PROJECT fv3-jedi      GIT "https://github.com/jcsda-internal/fv3-jedi.git"      BRANCH feature/ufs_dom UPDATE )  # updated from develop Dec 12 2023
-  if(UFS_APP MATCHES "^(S2S)$")
-    ecbuild_bundle( PROJECT soca        GIT "https://github.com/jcsda-internal/soca.git"          BRANCH feature/ufs_dom UPDATE )  # updated from develop Dec 12 2023
-    add_dependencies(soca ufs-weather-model)
-  endif()
+  ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop ) 
+  ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE )  
 elseif(UFS_APP MATCHES "^(NG-GODAS)$")
-  ecbuild_bundle( PROJECT soca     GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom UPDATE )  # updated from develop Dec 12 2023
+  ecbuild_bundle( PROJECT soca     GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom UPDATE )  
   add_dependencies(soca ufs-weather-model)
 else()
   message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}")

diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ Currently Loaded Modules:
  18) netcdf-c/4.9.2            39) bufr/12.0.0              60) udunits/2.2.28     81) py-python-dateutil/2.8.2    102) sigio/2.3.2
  19) netcdf-fortran/4.6.0      40) git-lfs/3.1.4            61) ncview/2.1.8       82) py-pytz/2022.2.1            103) w3emc/2.9.2
  20) parallel-netcdf/1.12.2    41) ecbuild/3.7.2            62) netcdf-cxx4/4.3.1  83) py-pandas/1.4.0             104) jedi-ufs-env/unified-dev
- 21) parallelio/2.5.9          42) openjpeg/2.3.1           63) json/3.10.5        84) py-pybind11/2.8.1           105) [email protected]
+ 21) parallelio/2.5.9          42) openjpeg/2.3.1           63) json/3.10.5        84) py-pybind11/2.8.1           105) fms@2023.02.01
 ```
 On a pre-configured platform that has spack-stack-1.4.1 (or 1.4.0) installed, load the spack-stack modules as described in https://spack-stack.readthedocs.io/en/1.4.1 (https://spack-stack.readthedocs.io/en/1.4.0), followed by `module load jedi-ufs-env`. On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.4.1 to build the `skylab-dev` or `unified-dev` environment, then run (use the appropriate path, compiler/mpi/python versions for your system):
 ```
@@ -38,6 +38,7 @@ module load stack-apple-clang/13.1.6
 module load stack-openmpi/4.1.5
 module load stack-python/3.10.8
 module load jedi-ufs-env/unified-dev
+module load fms/2023.02.01
 ```
 Further, for testing of the newest `[email protected]` tag, verify if the platform you are on has this version available and if yes, run
 ```