diff --git a/CMakeLists.txt b/CMakeLists.txt index 07c1719..d8f18fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,7 @@ set(FV3_FORECAST_MODEL "UFS") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH feature/ufs_dom UPDATE ) # updated from develop Dec 05 2023 +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) message(INFO "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}") include_directories(${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_INCLUDEDIR}) include_directories(${DEPEND_LIB_ROOT}/include_r8) @@ -171,10 +171,10 @@ set_target_properties( mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CM ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.2.0 ) if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(ATM)$") option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE ) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom UPDATE ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom UPDATE ) add_dependencies(soca ufs-weather-model) else() message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") diff --git a/README.md b/README.md index ceaed45..0ca8e64 100644 --- a/README.md +++ b/README.md @@ -4,47 +4,35 @@ Bundle for interfacing UFS models with JEDI interfaces ## Required thirdparty libraries -This bundle requires the following spack-stack modules be loaded (all except `fms@2022.04` are in the `skylab-3.0.0` environment): +This bundle requires loading the `jedi-ufs-env` and after that the `fms/2023.02.01` modules of spack-stack-1.5.1, or any equivalent installation providing the following modules: ``` > module li Currently Loaded Modules: - 1) stack-apple-clang/13.1.6 22) esmf/8.4.2 43) eccodes/2.27.0 64) json-schema-validator/2.1.0 85) py-pycodestyle/2.8.0 - 2) pmix/4.2.3 23) llvm-openmp/16.0.0 44) eigen/3.4.0 65) odc/1.4.6 86) py-pyhdf/0.10.4 - 3) zlib/1.2.13 24) libjpeg/2.1.0 45) openblas/0.3.19 66) py-attrs/22.2.0 87) libyaml/0.2.5 - 4) openmpi/4.1.5 25) jasper/2.0.32 46) eckit/1.23.1 67) py-pycparser/2.21 88) py-pyyaml/6.0 - 5) stack-openmpi/4.1.5 26) libpng/1.6.37 47) fftw/3.3.10 68) py-cffi/1.15.1 89) py-gast/0.5.3 - 6) pigz/2.7 27) g2/3.4.5 48) fckit/0.10.1 69) py-findlibs/0.0.2 90) py-beniget/0.4.1 - 7) zstd/1.5.2 28) g2tmpl/1.10.2 49) fiat/1.1.0 70) py-setuptools/59.4.0 91) py-ply/3.11 - 8) tar/1.34 29) sp/2.3.3 50) ectrans/1.2.0 71) py-numpy/1.22.3 92) py-pythran/0.12.2 - 9) gettext/0.21.1 30) ip/3.3.3 51) atlas/0.33.0 72) py-eccodes/1.4.2 93) py-scipy/1.9.3 - 10) libxcrypt/4.4.33 31) cmake/3.23.1 52) gsibec/1.1.2 73) py-f90nml/1.4.3 94) py-xarray/2022.3.0 - 11) sqlite/3.40.1 32) git/2.36.0 53) gsl-lite/0.37.0 74) py-h5py/3.7.0 95) jedi-base-env/1.0.0 - 12) python/3.10.8 33) libbacktrace/2020-02-19 54) hdf/4.2.15 75) py-cftime/1.0.3.4 96) gftl/1.8.3 - 13) stack-python/3.10.8 34) nccmp/1.9.0.1 55) jedi-cmake/1.4.0 76) py-netcdf4/1.5.3 97) gftl-shared/1.5.0 - 14) bacio/2.4.1 35) py-pip/23.0 56) libxt/1.1.5 77) py-bottleneck/1.3.5 98) yafyaml/0.5.1 - 15) curl/8.0.1 36) wget/1.21.3 57) libxmu/1.1.2 78) py-packaging/23.0 99) mapl/2.35.2-esmf-8.4.2 - 16) pkg-config/0.29.2 37) base-env/1.0.0 58) libxpm/3.5.12 79) py-numexpr/2.8.3 100) w3emc/2.10.0 - 17) hdf5/1.14.0 38) boost/1.78.0 59) libxaw/1.0.13 80) py-six/1.16.0 101) nemsio/2.5.2 - 18) netcdf-c/4.9.2 39) bufr/12.0.0 60) udunits/2.2.28 81) py-python-dateutil/2.8.2 102) sigio/2.3.2 - 19) netcdf-fortran/4.6.0 40) git-lfs/3.1.4 61) ncview/2.1.8 82) py-pytz/2022.2.1 103) w3emc/2.9.2 - 20) parallel-netcdf/1.12.2 41) ecbuild/3.7.2 62) netcdf-cxx4/4.3.1 83) py-pandas/1.4.0 104) jedi-ufs-env/unified-dev - 21) parallelio/2.5.9 42) openjpeg/2.3.1 63) json/3.10.5 84) py-pybind11/2.8.1 105) fms@2023.02.01 -``` -On a pre-configured platform that has spack-stack-1.4.1 (or 1.4.0) installed, load the spack-stack modules as described in https://spack-stack.readthedocs.io/en/1.4.1 (https://spack-stack.readthedocs.io/en/1.4.0), followed by `module load jedi-ufs-env`. On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.4.1 to build the `skylab-dev` or `unified-dev` environment, then run (use the appropriate path, compiler/mpi/python versions for your system): -``` -module use /Users/heinzell/prod/spack-stack-v1/envs/unified-dev/install/modulefiles/Core + 1) stack-apple-clang/13.1.6 15) snappy/1.1.10 29) libpng/1.6.37 43) ecbuild/3.7.2 57) hdf/4.2.15 71) py-cffi/1.15.1 85) py-six/1.16.0 99) gftl-shared/1.6.1 + 2) pmix/4.2.3 16) c-blosc/1.21.4 30) g2/3.4.5 44) openjpeg/2.3.1 58) jedi-cmake/1.4.0 72) py-findlibs/0.0.2 86) py-python-dateutil/2.8.2 100) fargparse/1.5.0 + 3) zlib/1.2.13 17) curl/8.1.2 31) g2tmpl/1.10.2 45) eccodes/2.27.0 59) libxt/1.1.5 73) py-setuptools/59.4.0 87) py-pytz/2023.3 101) mapl/2.40.3-esmf-8.5.0 + 4) openmpi/4.1.5 18) pkg-config/0.29.2 32) sp/2.3.3 46) eigen/3.4.0 60) libxmu/1.1.4 74) py-numpy/1.22.3 88) py-pandas/1.5.3 102) w3emc/2.10.0 + 5) stack-openmpi/4.1.5 19) hdf5/1.14.0 33) ip/4.3.0 47) openblas/0.3.19 61) libxpm/3.5.12 75) py-eccodes/1.4.2 89) py-pybind11/2.8.1 103) nemsio/2.5.4 + 6) pigz/2.7 20) netcdf-c/4.9.2 34) cmake/3.27.0 48) eckit/1.24.4 62) libxaw/1.0.13 76) py-f90nml/1.4.3 90) py-pycodestyle/2.8.0 104) sfcio/1.4.1 + 7) zstd/1.5.2 21) netcdf-fortran/4.6.0 35) git/2.32.0 49) fftw/3.3.10 63) udunits/2.2.28 77) py-h5py/3.7.0 91) py-pyhdf/0.10.4 105) sigio/2.3.2 + 8) tar/1.34 22) parallel-netcdf/1.12.2 36) libbacktrace/2020-02-19 50) fckit/0.11.0 64) ncview/2.1.8 78) py-cftime/1.0.3.4 92) libyaml/0.2.5 106) w3nco/2.4.1 + 9) gettext/0.21.1 23) parallelio/2.5.10 37) nccmp/1.9.0.1 51) fiat/1.2.0 65) netcdf-cxx4/4.3.1 79) py-netcdf4/1.5.8 93) py-pyyaml/5.4.1 107) jedi-ufs-env/1.0.0 + 10) libxcrypt/4.4.35 24) esmf/8.5.0 38) py-pip/23.1.2 52) ectrans/1.2.0 66) json/3.10.5 80) py-bottleneck/1.3.7 94) py-scipy/1.9.3 + 11) sqlite/3.42.0 25) llvm-openmp/16.0.0 39) wget/1.21.2 53) atlas/0.35.0 67) json-schema-validator/2.1.0 81) py-numexpr/2.8.4 95) py-packaging/23.1 + 12) python/3.10.8 26) fms/2023.02.01 40) base-env/1.0.0 54) git-lfs/3.3.0 68) odc/1.4.6 82) py-et-xmlfile/1.0.1 96) py-xarray/2022.3.0 + 13) stack-python/3.10.8 27) libjpeg/2.1.0 41) boost/1.78.0 55) gsibec/1.1.3 69) py-attrs/21.4.0 83) py-jdcal/1.3 97) jedi-base-env/1.0.0 + 14) bacio/2.4.1 28) jasper/2.0.32 42) bufr/12.0.1 56) gsl-lite/0.37.0 70) py-pycparser/2.21 84) py-openpyxl/3.0.7 98) gftl/1.10.0 +``` +On a pre-configured platform that has spack-stack-1.5.1 installed, load the spack-stack modules as described in https://spack-stack.readthedocs.io/en/1.5.1, followed by `module load jedi-ufs-env`. On a configurable (user) platform, follow the instructions in https://spack-stack.readthedocs.io/en/1.5.1 to build the `skylab-dev` or `unified-dev` environment, then run (use the appropriate path, compiler/mpi/python versions for your system): +``` +module use /Users/heinzell/prod/spack-stack-1.5.1/envs/unified-dev/install/modulefiles/Core module load stack-apple-clang/13.1.6 module load stack-openmpi/4.1.5 module load stack-python/3.10.8 module load jedi-ufs-env/unified-dev module load fms/2023.02.01 ``` -Further, for testing of the newest `fms@2023.02-beta1` tag, verify if the platform you are on has this version available and if yes, run -``` -# Loading fms/2023.02-beta1 replaces the default fms@2023.01 -module load fms/2023.02-beta1 -``` ## Building ufs-bundle @@ -59,39 +47,15 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DUFS_APP=ATMAERO .. 2>&1 | tee log.cmake cmake -DCMAKE_BUILD_TYPE=Debug -DUFS_APP=NG-GODAS .. 2>&1 | tee log.cmake cmake -DCMAKE_BUILD_TYPE=Debug -DUFS_APP=S2S .. 2>&1 | tee log.cmake ``` -When using the native Apple `clang` compiler on macOS with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="${llvm_openmp_ROOT}/lib/libomp.dylib"` to the `cmake` command. - -macOS users may also encounter this error during the linker phase: -``` -dyld[47628]: dyld[47619]: Library not loaded: 'libpioc.dylib' - Referenced from: '/Users/hLibrary not loaded: 'libpioc.dyleinzell/work/ufs-bundle/20230714ib' -... -``` -This error comes from the `pioc` library not being installed correctly (a problem we have raised to the `parallelio` maintainers): -``` -> otool -L ${parallelio_ROOT}/lib/libpioc.dylib -/Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/parallelio-2.5.9-r5txd2q/lib/libpioc.dylib: - libpioc.dylib (compatibility version 0.0.0, current version 0.0.0) - /Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/netcdf-c-4.9.2-vrrvi2u/lib/libnetcdf.19.dylib (compatibility version 22.0.0, current version 22.2.0) - /Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/parallel-netcdf-1.12.2-mfx2uut/lib/libpnetcdf.4.dylib (compatibility version 5.0.0, current version 5.2.0) - /Users/heinzell/prod/spack-stack-1.4.1/envs/unified-env/install/apple-clang/13.1.6/openmpi-4.1.5-j7pjg6h/lib/libmpi.40.dylib (compatibility version 71.0.0, current version 71.5.0) - /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.100.3) -``` -The first identifier to the library itself must either contain `@rpath/` and the `RPATH` needs to be added correctly, or (easier) it must contain the full path to the library. This can be fixed by running: -``` -install_name_tool -id ${parallelio_ROOT}/lib/libpioc.dylib ${parallelio_ROOT}/lib/libpioc.dylib -``` -A similar fix may be needed for `libpiof.dylib`, depending the `UFS_APP`: -``` -install_name_tool -id ${parallelio_ROOT}/lib/libpiof.dylib ${parallelio_ROOT}/lib/libpiof.dylib -install_name_tool -change libpioc.dylib ${parallelio_ROOT}/lib/libpioc.dylib ${parallelio_ROOT}/lib/libpiof.dylib -``` +When using the native Apple `clang` compiler on macOS with `llvm-openmp` installed via homebrew or spack, it may be necessary to add `-DCMAKE_SHARED_LINKER_FLAGS="${llvm_openmp_ROOT}/lib/libomp.dylib"` to the `cmake` command. We have identified this beig an issue with FMS not declaring its OpenMP dependencies correctly. A bug fix was merged for FMS recently, once the next tag finds its way into spack-stack this will no longer be needed. While building with soca (NG-GODAS or S2S), there will be a long pause during configuration when `cmake` is downloading the input files for the test to be run. After configuration, run `make -j 8` to build. The ctests for ATM all have `_ufs` in their names. After running `ctest -R get_`, run the following: + +# DH 20240708 - NOTE: EVERYTHING BELOW IS OUTDATED - UPDATE ONCE UFS TESTS ARE FIXED ON MACOS (WIP) ``` ctest -R _ufs 2>&1 | tee log.ctest.ufs Test project /Users/heinzell/work/ufs-bundle/20221114/build-debug-atm-20230121-with-ufs-import