heterogeneous-ci.yml

name: Heterogeneous Tests

on:
  push:
    branches: [ main, ci-fix ]
  pull_request:
    branches: [ main, ci-fix ]
  merge_group:
    branches: [ main, ci-fix ]

env:
  CUDA_HOME: /usr/local/cuda
  CUDACXX: nvcc
  MKLROOT: /opt/intel/oneapi/mkl/latest/
  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

jobs:
  test-heterogeneous:
    if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
    runs-on: [self-hosted, linux]
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: 'recursive'
    - name: Install dependencies
      run: |
        rm -f ~/.dace.conf
        rm -rf .dacecache tests/.dacecache
        python -m venv ~/.venv      # create venv so we can use pip
        source ~/.venv/bin/activate # activate venv
        python -m pip install --upgrade pip
        pip install flake8 pytest-xdist coverage
        pip install mpi4py pytest-mpi
        pip uninstall -y dace
        pip install -e ".[testing]"
        curl -Os https://uploader.codecov.io/latest/linux/codecov
        chmod +x codecov

    - name: Test dependencies
      run: |
        papi_avail

    - name: Run parallel pytest 
      run: |
        source ~/.venv/bin/activate # activate venv
        export DACE_cache=unique
        pytest --cov-report=xml --cov=dace --tb=short -m "verilator or mkl or papi or datainstrument"

    - name: Run MPI tests
      run: |
        export NOSTATUSBAR=1
        export DACE_cache=single
        export COVERAGE_RCFILE=`pwd`/.coveragerc
        export PYTHON_BINARY="coverage run --source=dace --parallel-mode"
        source ~/.venv/bin/activate # activate venv
        ./tests/mpi_test.sh

        
    - name: Test MPI with pytest
      run: |
        export NOSTATUSBAR=1
        export DACE_testing_serialization=1
        export DACE_testing_deserialize_exception=1
        export DACE_cache=unique
        source ~/.venv/bin/activate # activate venv
        mpirun -n 2 coverage run --source=dace --parallel-mode -m pytest -x --with-mpi --tb=short -m "mpi"
    
    - name: Test ScaLAPACK PBLAS with pytest
      run: |
        export NOSTATUSBAR=1
        export DACE_testing_serialization=1
        export DACE_testing_deserialize_exception=1
        export DACE_cache=unique
        export DACE_library_pblas_default_implementation=ReferenceOpenMPI
        source ~/.venv/bin/activate # activate venv
        for i in {1..4}
        do
          mpirun -n "$i" --oversubscribe coverage run --source=dace --parallel-mode -m pytest -x --with-mpi --tb=short -m "scalapack"
        done

    - name: Report overall coverage
      run: |
        export COVERAGE_RCFILE=`pwd`/.coveragerc
        source ~/.venv/bin/activate # activate venv
        coverage combine . */; coverage report; coverage xml
        reachable=0
        ping -W 2 -c 1 codecov.io || reachable=$?
        if [ $reachable -eq 0 ]; then
          ./codecov
        else
          echo "Codecov.io is unreachable"
        fi