Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use develop branch of fv3-jedi-linearmodel; update README.md to use [email protected] #51

Merged
merged 1 commit into from
Feb 8, 2024
Merged

Use develop branch of fv3-jedi-linearmodel; update README.md to use [email protected] #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ set(FV3_FORECAST_MODEL "UFS")

# fv3-jedi linear model
# ---------------------
ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom UPDATE ) # updated from develop Dec 05 2023
ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH develop UPDATE )
message(INFO "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
include_directories(${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_INCLUDEDIR})
include_directories(${DEPEND_LIB_ROOT}/include_r8)
Expand Down Expand Up @@ -171,10 +171,10 @@ set_target_properties( mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CM
ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 )
if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(ATM)$")
option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE )
elseif(UFS_APP MATCHES "^(NG-GODAS)$")
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom UPDATE )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom UPDATE )
add_dependencies(soca ufs-weather-model)
else()
message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}")
Expand Down
80 changes: 22 additions & 58 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,35 @@ Bundle for interfacing UFS models with JEDI interfaces

## Required thirdparty libraries

This bundle requires the following spack-stack modules be loaded (all except `fms@2022.04` are in the `skylab-3.0.0` environment):
This bundle requires loading the `jedi-ufs-env` and after that the `fms/2023.02.01` modules of spack-stack-1.5.1, or any equivalent installation providing the following modules:
```
> module li

Currently Loaded Modules:
1) stack-apple-clang/13.1.6 22) esmf/8.4.2 43) eccodes/2.27.0 64) json-schema-validator/2.1.0 85) py-pycodestyle/2.8.0
2) pmix/4.2.3 23) llvm-openmp/16.0.0 44) eigen/3.4.0 65) odc/1.4.6 86) py-pyhdf/0.10.4
3) zlib/1.2.13 24) libjpeg/2.1.0 45) openblas/0.3.19 66) py-attrs/22.2.0 87) libyaml/0.2.5
4) openmpi/4.1.5 25) jasper/2.0.32 46) eckit/1.23.1 67) py-pycparser/2.21 88) py-pyyaml/6.0
5) stack-openmpi/4.1.5 26) libpng/1.6.37 47) fftw/3.3.10 68) py-cffi/1.15.1 89) py-gast/0.5.3
6) pigz/2.7 27) g2/3.4.5 48) fckit/0.10.1 69) py-findlibs/0.0.2 90) py-beniget/0.4.1
7) zstd/1.5.2 28) g2tmpl/1.10.2 49) fiat/1.1.0 70) py-setuptools/59.4.0 91) py-ply/3.11
8) tar/1.34 29) sp/2.3.3 50) ectrans/1.2.0 71) py-numpy/1.22.3 92) py-pythran/0.12.2
9) gettext/0.21.1 30) ip/3.3.3 51) atlas/0.33.0 72) py-eccodes/1.4.2 93) py-scipy/1.9.3
10) libxcrypt/4.4.33 31) cmake/3.23.1 52) gsibec/1.1.2 73) py-f90nml/1.4.3 94) py-xarray/2022.3.0
11) sqlite/3.40.1 32) git/2.36.0 53) gsl-lite/0.37.0 74) py-h5py/3.7.0 95) jedi-base-env/1.0.0
12) python/3.10.8 33) libbacktrace/2020-02-19 54) hdf/4.2.15 75) py-cftime/1.0.3.4 96) gftl/1.8.3
13) stack-python/3.10.8 34) nccmp/1.9.0.1 55) jedi-cmake/1.4.0 76) py-netcdf4/1.5.3 97) gftl-shared/1.5.0
14) bacio/2.4.1 35) py-pip/23.0 56) libxt/1.1.5 77) py-bottleneck/1.3.5 98) yafyaml/0.5.1
15) curl/8.0.1 36) wget/1.21.3 57) libxmu/1.1.2 78) py-packaging/23.0 99) mapl/2.35.2-esmf-8.4.2
16) pkg-config/0.29.2 37) base-env/1.0.0 58) libxpm/3.5.12 79) py-numexpr/2.8.3 100) w3emc/2.10.0
17) hdf5/1.14.0 38) boost/1.78.0 59) libxaw/1.0.13 80) py-six/1.16.0 101) nemsio/2.5.2
18) netcdf-c/4.9.2 39) bufr/12.0.0 60) udunits/2.2.28 81) py-python-dateutil/2.8.2 102) sigio/2.3.2
19) netcdf-fortran/4.6.0 40) git-lfs/3.1.4 61) ncview/2.1.8 82) py-pytz/2022.2.1 103) w3emc/2.9.2
20) parallel-netcdf/1.12.2 41) ecbuild/3.7.2 62) netcdf-cxx4/4.3.1 83) py-pandas/1.4.0 104) jedi-ufs-env/unified-dev
21) parallelio/2.5.9 42) openjpeg/2.3.1 63) json/3.10.5 84) py-pybind11/2.8.1 105) [email protected]
```
On a pre-configured platform that has spack-stack-1.4.1 (or 1.4.0) installed, load the spack-stack modules as described in https://spack-stack.readthedocs.io/en/1.4.1 (https://spack-stack.readthedocs.io/en/1.4.0), followed by `module load jedi-ufs-env`. On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.4.1 to build the `skylab-dev` or `unified-dev` environment, then run (use the appropriate path, compiler/mpi/python versions for your system):
```
module use /Users/heinzell/prod/spack-stack-v1/envs/unified-dev/install/modulefiles/Core
1) stack-apple-clang/13.1.6 15) snappy/1.1.10 29) libpng/1.6.37 43) ecbuild/3.7.2 57) hdf/4.2.15 71) py-cffi/1.15.1 85) py-six/1.16.0 99) gftl-shared/1.6.1
2) pmix/4.2.3 16) c-blosc/1.21.4 30) g2/3.4.5 44) openjpeg/2.3.1 58) jedi-cmake/1.4.0 72) py-findlibs/0.0.2 86) py-python-dateutil/2.8.2 100) fargparse/1.5.0
3) zlib/1.2.13 17) curl/8.1.2 31) g2tmpl/1.10.2 45) eccodes/2.27.0 59) libxt/1.1.5 73) py-setuptools/59.4.0 87) py-pytz/2023.3 101) mapl/2.40.3-esmf-8.5.0
4) openmpi/4.1.5 18) pkg-config/0.29.2 32) sp/2.3.3 46) eigen/3.4.0 60) libxmu/1.1.4 74) py-numpy/1.22.3 88) py-pandas/1.5.3 102) w3emc/2.10.0
5) stack-openmpi/4.1.5 19) hdf5/1.14.0 33) ip/4.3.0 47) openblas/0.3.19 61) libxpm/3.5.12 75) py-eccodes/1.4.2 89) py-pybind11/2.8.1 103) nemsio/2.5.4
6) pigz/2.7 20) netcdf-c/4.9.2 34) cmake/3.27.0 48) eckit/1.24.4 62) libxaw/1.0.13 76) py-f90nml/1.4.3 90) py-pycodestyle/2.8.0 104) sfcio/1.4.1
7) zstd/1.5.2 21) netcdf-fortran/4.6.0 35) git/2.32.0 49) fftw/3.3.10 63) udunits/2.2.28 77) py-h5py/3.7.0 91) py-pyhdf/0.10.4 105) sigio/2.3.2
8) tar/1.34 22) parallel-netcdf/1.12.2 36) libbacktrace/2020-02-19 50) fckit/0.11.0 64) ncview/2.1.8 78) py-cftime/1.0.3.4 92) libyaml/0.2.5 106) w3nco/2.4.1
9) gettext/0.21.1 23) parallelio/2.5.10 37) nccmp/1.9.0.1 51) fiat/1.2.0 65) netcdf-cxx4/4.3.1 79) py-netcdf4/1.5.8 93) py-pyyaml/5.4.1 107) jedi-ufs-env/1.0.0
10) libxcrypt/4.4.35 24) esmf/8.5.0 38) py-pip/23.1.2 52) ectrans/1.2.0 66) json/3.10.5 80) py-bottleneck/1.3.7 94) py-scipy/1.9.3
11) sqlite/3.42.0 25) llvm-openmp/16.0.0 39) wget/1.21.2 53) atlas/0.35.0 67) json-schema-validator/2.1.0 81) py-numexpr/2.8.4 95) py-packaging/23.1
12) python/3.10.8 26) fms/2023.02.01 40) base-env/1.0.0 54) git-lfs/3.3.0 68) odc/1.4.6 82) py-et-xmlfile/1.0.1 96) py-xarray/2022.3.0
13) stack-python/3.10.8 27) libjpeg/2.1.0 41) boost/1.78.0 55) gsibec/1.1.3 69) py-attrs/21.4.0 83) py-jdcal/1.3 97) jedi-base-env/1.0.0
14) bacio/2.4.1 28) jasper/2.0.32 42) bufr/12.0.1 56) gsl-lite/0.37.0 70) py-pycparser/2.21 84) py-openpyxl/3.0.7 98) gftl/1.10.0
```
On a pre-configured platform that has spack-stack-1.5.1 installed, load the spack-stack modules as described in https://spack-stack.readthedocs.io/en/1.5.1, followed by `module load jedi-ufs-env`. On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.5.1 to build the `skylab-dev` or `unified-dev` environment, then run (use the appropriate path, compiler/mpi/python versions for your system):
```
module use /Users/heinzell/prod/spack-stack-1.5.1/envs/unified-dev/install/modulefiles/Core
module load stack-apple-clang/13.1.6
module load stack-openmpi/4.1.5
module load stack-python/3.10.8
module load jedi-ufs-env/unified-dev
module load fms/2023.02.01
```
Further, for testing of the newest `[email protected]` tag, verify if the platform you are on has this version available and if yes, run
```
# Loading fms/2023.02-beta1 replaces the default [email protected]
module load fms/2023.02-beta1
```

## Building ufs-bundle

Expand All @@ -59,39 +47,15 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DUFS_APP=ATMAERO .. 2>&1 | tee log.cmake
cmake -DCMAKE_BUILD_TYPE=Debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.cmake
cmake -DCMAKE_BUILD_TYPE=Debug -DUFS_APP=S2S .. 2>&1 | tee log.cmake
```
When using the native Apple `clang` compiler on macOS with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="${llvm_openmp_ROOT}/lib/libomp.dylib"` to the `cmake` command.

macOS users may also encounter this error during the linker phase:
```
dyld[47628]: dyld[47619]: Library not loaded: 'libpioc.dylib'
Referenced from: '/Users/hLibrary not loaded: 'libpioc.dyleinzell/work/ufs-bundle/20230714ib'
...
```
This error comes from the `pioc` library not being installed correctly (a problem we have raised to the `parallelio` maintainers):
```
> otool -L ${parallelio_ROOT}/lib/libpioc.dylib
/Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/parallelio-2.5.9-r5txd2q/lib/libpioc.dylib:
libpioc.dylib (compatibility version 0.0.0, current version 0.0.0)
/Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/netcdf-c-4.9.2-vrrvi2u/lib/libnetcdf.19.dylib (compatibility version 22.0.0, current version 22.2.0)
/Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/parallel-netcdf-1.12.2-mfx2uut/lib/libpnetcdf.4.dylib (compatibility version 5.0.0, current version 5.2.0)
/Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/openmpi-4.1.5-j7pjg6h/lib/libmpi.40.dylib (compatibility version 71.0.0, current version 71.5.0)
/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.100.3)
```
The first identifier to the library itself must either contain `@rpath/` and the `RPATH` needs to be added correctly, or (easier) it must contain the full path to the library. This can be fixed by running:
```
install_name_tool -id ${parallelio_ROOT}/lib/libpioc.dylib ${parallelio_ROOT}/lib/libpioc.dylib
```
A similar fix may be needed for `libpiof.dylib`, depending the `UFS_APP`:
```
install_name_tool -id ${parallelio_ROOT}/lib/libpiof.dylib ${parallelio_ROOT}/lib/libpiof.dylib
install_name_tool -change libpioc.dylib ${parallelio_ROOT}/lib/libpioc.dylib ${parallelio_ROOT}/lib/libpiof.dylib
```
When using the native Apple `clang` compiler on macOS with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="${llvm_openmp_ROOT}/lib/libomp.dylib"` to the `cmake` command. We have identified this beig an issue with FMS not declaring its OpenMP dependencies correctly. A bug fix was merged for FMS recently, once the next tag finds its way into spack-stack this will no longer be needed.

While building with soca (NG-GODAS or S2S), there will be a long pause during configuration when `cmake` is downloading the input files for the test to be run.

After configuration, run `make -j 8` to build.

The ctests for ATM all have `_ufs` in their names. After running `ctest -R get_`, run the following:

# DH 20240708 - NOTE: EVERYTHING BELOW IS OUTDATED - UPDATE ONCE UFS TESTS ARE FIXED ON MACOS (WIP)
```
ctest -R _ufs 2>&1 | tee log.ctest.ufs
Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-atm-20230121-with-ufs-import
Expand Down
Loading