ci.yml

name: CI

on:
  push:
    branches:
      - 'devel*'
  pull_request:
    branches:
      - 'devel*'
  schedule:
    - cron:  '45 6 * * *'

jobs:

  linting:
    runs-on: ubuntu-latest
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - uses: actions/setup-python@v5
        with:
          python-version: 3.7
      - name: Install dependencies
        run: |
          python -m venv testenv
          . testenv/bin/activate
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r requirements-ci.txt
          radical-stack
      - name: Lint with flake8 and pylint
        run: |
          . testenv/bin/activate
          LOG=`git log -n 1 | grep Merge` || echo 'no merging' && exit 0
          NEW=`echo $LOG | cut -d ' ' -f2`
          OLD=`echo $LOG | cut -d ' ' -f3`
          (test -z "$NEW" || test -z "$OLD") && echo 'nothing to lint' && exit 0
          DIFF=`git diff --name-only --diff-filter=db $NEW..$OLD`
          FILTERED=$(for f in $DIFF; do echo $f; done | \
                     grep -v -e 'concepts/' -e '/archive/' | \
                     grep -e '\.py$' |  sed "s|^|$PWD/|")
          test -z "$FILTERED" && echo 'nothing to lint' && exit 0
          flake8 $FILTERED
          pylint $FILTERED

  tests:
    runs-on: ubuntu-latest
    continue-on-error: true
    strategy:
      matrix:
        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m venv testenv
          . testenv/bin/activate
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r requirements-ci.txt
          radical-stack
      - name: Unit tests
        run: |
          . testenv/bin/activate
          radical-stack
          coverage run --source=radical.pilot -m pytest -ra -vvv --showlocals tests/unit_tests/ tests/component_tests/
          coverage xml
      - uses: codecov/codecov-action@v4
        if: ${{ matrix.python-version == '3.7' }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          files: ./coverage.xml

  notebook_integration_test:
    strategy:
      matrix:
        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ]
    uses: ./.github/workflows/run-rp-notebook.yml
    with:
      python-version: ${{ matrix.python-version }}
      requirements-file: 'requirements-docs-ci.txt'
      notebook-name: 'getting_started.ipynb'

  analytics:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y texlive cm-super
          sudo apt install -y texlive-fonts-extra texlive-extra-utils dvipng
          sudo apt install -y texlive-fonts-recommended texlive-latex-extra
          python -m venv testenv
          . testenv/bin/activate
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r requirements-ci.txt
      - name: Analyse example session
        timeout-minutes: 5
        if: always()
        env:
          RADICAL_LOG_LVL: 'DEBUG'
          RADICAL_REPORT: 'TRUE'
        run: |
          . testenv/bin/activate
          ./examples/00_getting_started.py local.localhost
          SID=$(find . -name "rp.session*" -type d -exec basename {} "rp.session" \;)
          echo "SID: $SID"
          radical-analytics-inspect "$SID"
          mkdir artifacts
          ls -la
          cp -R *.png *.stats artifacts
      - name: Collect session
        if: always()
        run: |
          SIDCLIENT=$(find . -name "rp.session*" -type d -exec basename {} "rp.session" \;)
          SIDAGENT="$HOME/radical.pilot.sandbox/$SIDCLIENT"
          echo "SIDCLIENT : $SIDCLIENT"
          echo "SIDAGENT  : $SIDAGENT"
          CLIENTNAME="analytics_client_$SIDCLIENT"
          AGENTNAME="analytics_agent_$SIDCLIENT"
          mkdir sessions_analytics
          # tar cvfj $CLIENTNAME.tar.bz2 $SIDCLIENT
          tar cvfj $CLIENTNAME.tar.bz2 `find $SIDCLIENT -type d -print`
          cp -R $CLIENTNAME.tar.bz2 sessions_analytics
          if [ -d "$SIDAGENT" ]; then
            tar cvfj $AGENTNAME.tar.bz2 `find $SIDAGENT -type d -print`
            cp -R $AGENTNAME.tar.bz2 sessions_analytics
          fi
      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v3
        with:
            name: artifacts
            path: artifacts
            retention-days: 5
      - name: Upload sessions
        if: always()
        uses: actions/upload-artifact@v3
        with:
            name: sessions_analytics
            path: sessions_analytics
            retention-days: 5

  examples:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.12' ]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y openmpi-bin libopenmpi-dev
          sudo apt install -y texlive cm-super
          sudo apt install -y texlive-fonts-extra texlive-extra-utils dvipng
          sudo apt install -y texlive-fonts-recommended texlive-latex-extra
          python -m venv testenv
          . testenv/bin/activate
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r requirements-ci.txt
          radical-stack
      - name: run examples
        timeout-minutes: 10
        run: |
          export RADICAL_LOG_LVL=DEBUG_9
          export RADICAL_PROFILE=True
          export RADICAL_DEBUG=TRUE
          export RADICAL_DEBUG_HELPER=TRUE
          export RADICAL_REPORT=TRUE
          export RADICAL_UTILS_ZMQ_LOG_LVL=INFO
          export RADICAL_UTILS_HEARTBEAT_LOG_LVL=INFO
          . testenv/bin/activate
          export RP_ROOT=$(pwd)
          export BASE=/home/runner/radical.pilot.sandbox/
          mkdir -p $BASE/client_sessions
          cd $BASE/client_sessions
          radical-stack
          for f in $RP_ROOT/examples/[01]*.py; do echo "=== $f"; $f; done

      - name: prepare example_artifacts
        if:   always()
        run: |
            tar zcf /home/runner/example_artifacts.tgz /home/runner/radical.pilot.sandbox/

      - name: upload example_artifacts
        if:   always()
        uses: actions/upload-artifact@v3
        with:
            name: example_artifacts
            path: /home/runner/example_artifacts.tgz