nvidia_docker.yml

name: Nvidia Container

on: [pull_request,workflow_dispatch]

jobs:

  build_scm:
    # The type of runner that the job will run on
    runs-on: ubuntu-22.04
    container:
      image: nvcr.io/nvidia/nvhpc:24.9-devel-cuda12.6-ubuntu22.04
    strategy:
      fail-fast: false
      matrix:
        fortran-compiler: [nvfortran]
        build-type:       [Release]#, Debug]
        enable-gpu-acc:   [False, True]
        py-version:       [3.9.12]

    # Environmental variables
    env:
      NETCDF:      /home/runner/netcdf
      NETCDF_VERSION: 4.9.2
      NETCDFF_VERSION: 4.6.1
      bacio_ROOT:  /home/runner/bacio
      sp_ROOT:     /home/runner/NCEPLIBS-sp
      w3emc_ROOT:  /home/runner/myw3emc
      SCM_ROOT:    /home/runner/work/ccpp-scm/ccpp-scm
      HDF5_ROOT:   /home/runner/hdf5
      HDF5_VERSION: 1.14.5
      MPI_ROOT:    /home/runner/openmpi
      MPI_VERSION: 4.1.7
      suites:      SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0
      suites_ps:   SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps

    # Workflow steps
    steps:

    #######################################################################################
    # Cleanup space
    #######################################################################################
    - name: Check space (pre)
      run: |
        df -h

    - name: Free Disk Space (Ubuntu)
      uses: jlumbroso/free-disk-space@main
      with:
        # this might remove tools that are actually needed,
        # if set to "true" but frees about 6 GB
        tool-cache: false

        # all of these default to true, but feel free to set to
        # "false" if necessary for your workflow
        android: false
        dotnet: false
        haskell: true
        large-packages: true
        docker-images: false
        swap-storage: false

    - name: Check space (post)
      run: |
        df -h

    #######################################################################################
    # Initial
    #######################################################################################
    - name: Checkout SCM code (into /home/runner/work/ccpp-scm/)
      uses: actions/checkout@v4

    - name: Set Git Safe Directory
      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

    - name: Initialize submodules
      run: git submodule update --init --recursive

    #######################################################################################
    # Python setup
    #######################################################################################
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{matrix.py-version}}

    - name: Add Python libraries
      run: |
        apt-get update
        apt-get -y install python3-pip python3.11-venv python3-netcdf4 curl wget libcurl4-openssl-dev libxml2
        apt-get -y install python3-numpy

    - name: Pip install and check
      run: |
        which pip
        pip install f90nml
        python3 -c "import f90nml"

    - name: Check for nvfortran
      run: |
        which nvfortran
        nvfortran --version

    #######################################################################################
    # Install Nvidia.
    #######################################################################################

    # - name: Nvidia setup compilers.
    #   env:
    #     NVCOMPILERS: /home/runner/hpc_sdk
    #     NVARCH: Linux_x86_64
    #     NVHPC_SILENT: true
    #     NVHPC_INSTALL_DIR: /home/runner/hpc_sdk
    #     NVHPC_INSTALL_TYPE: network
    #     NVHPC_INSTALL_LOCAL_DIR: /home/runner/hpc_sdk
    #   run: |
    #     mkdir /home/runner/hpc_sdk && cd /home/runner/hpc_sdk
    #     wget -q https://developer.download.nvidia.com/hpc-sdk/24.1/nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz
    #     tar xpzf nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz
    #     nvhpc_2024_241_Linux_x86_64_cuda_12.3/install
    #     export PATH=${PATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/bin
    #     export MANPATH=${MANPATH}:${NVCOMPILERS}/${NVARCH}/24.1/compilers/man
    #     echo "The nvfortran installed is:"
    #     nvfortran --version
    #     echo "The path to nvfortran is:"
    #     command -v nvfortran
    #     echo "Removing tarball"
    #     rm nvhpc_2024_241_Linux_x86_64_cuda_12.3.tar.gz

    # - name: Set environment for Nvidia compiler.
    #   run: |
    #     echo "CC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV
    #     echo "FC=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV
    #     echo "CMAKE_C_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvc" >> $GITHUB_ENV
    #     echo "CMAKE_Fortran_COMPILER=/home/runner/hpc_sdk/Linux_x86_64/24.1/compilers/bin/nvfortran" >> $GITHUB_ENV

    #######################################################################################
    # Install FORTRAN dependencies
    #######################################################################################

    # - name: Install Curl and zlib
    #   run: |
    #     sudo apt-get update
    #     sudo apt-get install curl
    #     sudo apt-get install libssl-dev libcurl4-openssl-dev
    #     sudo apt-get install zlib1g-dev

    - name: Cache HDF5
      id: cache-hdf5
      uses: actions/cache@v4
      with:
        path: /home/runner/hdf5
        KEY: cache-hdf5-${{matrix.fortran-compiler}}-key

    - name: Install HDF5
      if: steps.cache-hdf5.outputs.cache-hit != 'true'
      run: |
        wget -q https://github.com/HDFGroup/hdf5/releases/download/hdf5_${HDF5_VERSION}/hdf5-${HDF5_VERSION}.tar.gz
        tar zxf hdf5-${HDF5_VERSION}.tar.gz
        cd hdf5-${HDF5_VERSION}
        ./configure --prefix=${HDF5_ROOT}
        make -j
        make install

    - name: Setup HDF5 Paths
      run: |
        echo "LD_LIBRARY_PATH=$HDF5_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
        echo "PATH=$HDF5_ROOT/bin:$PATH" >> $GITHUB_ENV

    - name: Cache OpenMPI
      id: cache-openmpi
      uses: actions/cache@v4
      with:
        path: /home/runner/openmpi
        KEY: cache-openmpi-${{matrix.fortran-compiler}}-key

    - name: Configure OpenMPI
      if: steps.cache-openmpi.outputs.cache-hit != 'true'
      run: |
        cd ${HOME}
        wget -q https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${MPI_VERSION}.tar.gz
        tar zxf openmpi-${MPI_VERSION}.tar.gz
        cd openmpi-${MPI_VERSION}
        CFLAGS=-fPIC CXXFLAGS=-fPIC FCFLAGS=-fPIC ./configure --prefix=${MPI_ROOT}

    - name: Build OpenMPI
      if: steps.cache-openmpi.outputs.cache-hit != 'true'
      run: |
        cd ${HOME}/openmpi-${MPI_VERSION}
        make -j
        make install -j

    - name: Setup OpenMPI Paths
      run: |
        echo "PATH=${MPI_ROOT}/bin:$PATH" >> $GITHUB_ENV
        echo "LD_LIBRARY_PATH=${MPI_ROOT}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV

    - name: Check MPI Version
      run: |
        mpif90 --version

    - name: Set environment for Nvidia compiler with MPI
      run: |
        echo "CC=$(which mpicc)" >> $GITHUB_ENV
        echo "FC=$(which mpif90)" >> $GITHUB_ENV
        echo "CMAKE_C_COMPILER=$(which mpicc)" >> $GITHUB_ENV
        echo "CMAKE_Fortran_COMPILER=$(which mpif90)" >> $GITHUB_ENV

    - name: Check space (post)
      run: |
        df -h

    - name: Cache NetCDF library
      id: cache-netcdf
      uses: actions/cache@v4
      with:
        path: /home/runner/netcdf
        key: cache-netcdf-${{matrix.fortran-compiler}}-key

    - name: Setup NetCDF Paths
      run: |
        echo "LD_LIBRARY_PATH=$NETCDF/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
        echo "PATH=$NETCDF/bin:$PATH" >> $GITHUB_ENV

    # requires libxml2
    - name: Install NetCDF C library
      if: steps.cache-netcdf.outputs.cache-hit != 'true'
      run: |
        wget -q https://github.com/Unidata/netcdf-c/archive/refs/tags/v${NETCDF_VERSION}.tar.gz
        tar zxf v${NETCDF_VERSION}.tar.gz
        cd netcdf-c-${NETCDF_VERSION}
        CPPFLAGS="-I${HDF5_ROOT}/include" LDFLAGS="-L${HDF5_ROOT}/lib" ./configure --prefix=${NETCDF}
        make -j
        make install -j

    - name: Install NetCDF Fortran library
      if: steps.cache-netcdf.outputs.cache-hit != 'true'
      run: |
        wget -q https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${NETCDFF_VERSION}.tar.gz
        tar zxf v${NETCDFF_VERSION}.tar.gz
        cd netcdf-fortran-${NETCDFF_VERSION}
        FCFLAGS="-fPIC" FFLAGS="-fPIC" CPPFLAGS="-I${HDF5_ROOT}/include -I${NETCDF}/include" LDFLAGS="-L${HDF5_ROOT}/lib -L${NETCDF}/lib" ./configure --prefix=${NETCDF}
        make -j
        make install -j

    - name: Cache bacio library v2.4.1
      id: cache-bacio-fortran
      uses: actions/cache@v4
      with:
        path: /home/runner/bacio
        key: cache-bacio-fortran-${{matrix.fortran-compiler}}-key

    - name: Install bacio library v2.4.1
      if: steps.cache-bacio-fortran.outputs.cache-hit != 'true'
      run: |
        git clone --branch v2.4.1 https://github.com/NOAA-EMC/NCEPLIBS-bacio.git bacio
        cd bacio && mkdir build && cd build
        cmake -DCMAKE_INSTALL_PREFIX=${bacio_ROOT} ../
        make -j
        make install
        echo "bacio_DIR=/home/runner/bacio/lib/cmake/bacio" >> $GITHUB_ENV

    - name: Cache SP-library v2.3.3
      id: cache-sp-fortran
      uses: actions/cache@v4
      with:
        path: /home/runner/NCEPLIBS-sp
        key: cache-sp-fortran-${{matrix.fortran-compiler}}-key

    - name: Install SP-library v2.3.3
      if: steps.cache-sp-fortran.outputs.cache-hit != 'true'
      run: |
        git clone --branch v2.3.3 https://github.com/NOAA-EMC/NCEPLIBS-sp.git NCEPLIBS-sp
        cd NCEPLIBS-sp && mkdir build && cd build
        cmake -DCMAKE_INSTALL_PREFIX=${sp_ROOT} ../
        make -j
        make install
        echo "sp_DIR=/home/runner/NCEPLIBS-sp/lib/cmake/sp" >> $GITHUB_ENV

    - name: Cache w3emc library v2.9.2
      id: cache-w3emc-fortran
      uses: actions/cache@v4
      with:
        path: /home/runner/myw3emc
        key: cache-w3emc-fortran-${{matrix.fortran-compiler}}-key

    - name: Install w3emc library v2.9.2
      if: steps.cache-w3emc-fortran.outputs.cache-hit != 'true'
      run: |
        git clone --branch v2.9.2 https://github.com/NOAA-EMC/NCEPLIBS-w3emc.git NCEPLIBS-w3emc
        cd NCEPLIBS-w3emc && mkdir build && cd build
        cmake -DCMAKE_INSTALL_PREFIX=${w3emc_ROOT} ../
        make -j
        make install
        echo "w3emc_DIR=/home/runner/myw3emc/lib/cmake/w3emc" >> $GITHUB_ENV

    #######################################################################################
    # Build and run SCM regression tests (ccpp-scm/test/rt_test_cases.py)
    #######################################################################################

    - name: Check space (post)
      run: |
        df -h

    - name: Configure build with CMake
      run: |
        cd ${SCM_ROOT}/scm
        mkdir bin && cd bin
        cmake -DCCPP_SUITES=${suites},${suites_ps} -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -DENABLE_NVIDIA_OPENACC=${{matrix.enable-gpu-acc}} ../src

    - name: Build SCM.
      run: |
        cd ${SCM_ROOT}/scm/bin
        make -j

    - name: Check space (post)
      run: |
        df -h

    - name: Download data for SCM
      if: contains(matrix.enable-gpu-acc, 'False')
      run: |
        cd ${SCM_ROOT}
        ./contrib/get_all_static_data.sh
        ./contrib/get_thompson_tables.sh
        ./contrib/get_aerosol_climo.sh

    - name: Check space (post)
      run: |
        df -h

    - name: Run SCM RTs (w/o GPU)
      if: contains(matrix.enable-gpu-acc, 'False')
      run: |
        cd ${SCM_ROOT}/scm/bin
        ./run_scm.py --file /home/runner/work/ccpp-scm/ccpp-scm/test/rt_test_cases.py --runtime_mult 0.1 -v