refresh-notebooks.yaml

name: Refresh Notebooks
on:
  push:
  # workflow_dispatch:

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

env:
  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

jobs:
  start-runner-linux:
    name: Start EC2 runner
    runs-on: ubuntu-20.04
    outputs:
      label-38: ${{ steps.start-ec2-runner-38.outputs.label }}
      ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Start EC2 runner python 38
        id: start-ec2-runner-38
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        with:
          mode: start
          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
          ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
          ec2-instance-type: "m6i.metal"
          subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}

  refresh-notebooks:
    needs: [start-runner-linux]

    runs-on: ${{ needs.start-runner-linux.outputs.label-38 }}
    # Run in a clean container
    container:
      image: ubuntu:20.04
    defaults:
      run:
        shell: bash
    env:
      PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }}
      PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }}
      KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
      KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}

    steps:
      - name: Add masks
        run: |
          echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}"
          echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}"

      # Replace default archive.ubuntu.com from docker image with fr mirror
      # original archive showed performance issues and is farther away
      - name: Docker container related setup and git installation
        run: |
          TZ=Europe/Paris
          echo "TZ=${TZ}" >> "$GITHUB_ENV"
          ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
          sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
          apt update && apt install git git-lfs -y

      - name: Checkout Code
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
        with:
          lfs: true

      - name: 'Set up Python 3.8'
        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1
        with:
          python-version: '3.8'

      - name: Install dependencies
        run: |
          ./script/make_utils/setup_os_deps.sh
          make setup_env

      - name: Refresh Notebooks
        run: |
          make jupyter_execute

      - name: Prepare PR Body
        run: |
          SUCCESSFUL_NOTEBOOKS=$(cat ./successful_notebooks.txt)
          FAILED_NOTEBOOKS=$(cat ./failed_notebooks.txt)
          PR_BODY="Automatic PR with notebook refresh for ${{ github.ref_name }}.\n"
          PR_BODY+="## Successful Notebooks\n$SUCCESSFUL_NOTEBOOKS\n"
          PR_BODY+="## Failed Notebooks\n$FAILED_NOTEBOOKS"
          echo "PR_BODY=${PR_BODY}" >> "$GITHUB_ENV"

      - name: Open PR
        uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666
        with:
          token: ${{ secrets.BOT_TOKEN }}
          commit-message: "chore: refresh notebooks"
          branch: "refresh-notebooks-for-${{ github.ref_name }}"
          base: "${{ github.ref_name }}"
          title: "Refresh notebooks for ${{ github.ref_name }}"
          body: ${{ env.PR_BODY }}
          add-paths: |
            docs/**/*.ipynb

  stop-runner-linux:
    name: Stop EC2 runner
    needs: [refresh-notebooks, start-runner-linux]
    runs-on: ubuntu-20.04
    if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Stop EC2 runner python 38
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }}
        with:
          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
          label: ${{ needs.start-runner-linux.outputs.label-38 }}
          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }}
          mode: stop

  send-report:
    if: ${{ always() }}
    needs:
      [
        start-runner-linux,
        refresh-notebooks,
        stop-runner-linux,
      ]

    name: Send Slack notification
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608

      - name: Prepare whole job status
        if: ${{ always() }}
        continue-on-error: true
        env:
          NEEDS_JSON: ${{ toJSON(needs) }}
        run: |
          echo "${NEEDS_JSON}" > /tmp/needs_context.json
          JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \
          --needs_context_json /tmp/needs_context.json)
          echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV"

      - name: Slack Notification
        if: ${{ always() }}
        continue-on-error: true
        uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8
        env:
          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
          SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }}
          SLACK_MESSAGE: "Full run finished with status ${{ env.JOB_STATUS || 'failure' }} \
            (${{ env.ACTION_RUN_URL }})\n\
            - start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\
            - refresh-notebooks: ${{ needs.refresh-notebooks.result || 'Did not run.' }}\n\n\
            - stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}"
          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}