Refresh Notebooks #34
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Refresh Notebooks | |
| on: | |
| workflow_dispatch: | |
| concurrency: | |
| group: "${{ github.ref }}-${{ github.event_name }}-${{ github.workflow }}" | |
| cancel-in-progress: false | |
| env: | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| jobs: | |
| start-runner-linux: | |
| name: Start EC2 runner | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| label-38: ${{ steps.start-ec2-runner-38.outputs.label }} | |
| ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Start EC2 runner python 38 | |
| id: start-ec2-runner-38 | |
| uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} | |
| ec2-image-id: ${{ secrets.AWS_EC2_AMI }} | |
| ec2-instance-type: "m6i.metal" | |
| subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }} | |
| security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }} | |
| refresh-notebooks: | |
| needs: [start-runner-linux] | |
| runs-on: ${{ needs.start-runner-linux.outputs.label-38 }} | |
| # Run in a clean container | |
| container: | |
| image: ubuntu:20.04 | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }} | |
| PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }} | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} | |
| steps: | |
| - name: Add masks | |
| run: | | |
| echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}" | |
| # Replace default archive.ubuntu.com from docker image with fr mirror | |
| # original archive showed performance issues and is farther away | |
| - name: Docker container related setup and git installation | |
| run: | | |
| TZ=Europe/Paris | |
| echo "TZ=${TZ}" >> "$GITHUB_ENV" | |
| ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone | |
| sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list | |
| apt update && apt install git git-lfs -y | |
| - name: Checkout Code | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 | |
| with: | |
| lfs: true | |
| - name: 'Set up Python 3.8' | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d | |
| with: | |
| python-version: '3.8' | |
| - name: Install dependencies | |
| run: | | |
| ./script/make_utils/setup_os_deps.sh | |
| make setup_env | |
| - name: Refresh Notebooks | |
| run: | | |
| make jupyter_execute | |
| - name: Prepare PR Body | |
| run: | | |
| SUCCESSFUL_NOTEBOOKS=$(cat ./successful_notebooks.txt | tr '\n' ' ' | sed 's/ /\\n- /g') | |
| FAILED_NOTEBOOKS=$(cat ./failed_notebooks.txt | tr '\n' ' ' | sed 's/ /\\n- /g') | |
| PR_BODY="Automatic PR with notebook refresh for ${{ github.ref_name }}.\\n" | |
| PR_BODY+="## Successful Notebooks\\n- $SUCCESSFUL_NOTEBOOKS\\n" | |
| PR_BODY+="## Failed Notebooks\\n- $FAILED_NOTEBOOKS" | |
| echo "PR_BODY=${PR_BODY}" >> "$GITHUB_ENV" | |
| - name: Open PR | |
| uses: peter-evans/create-pull-request@6d6857d36972b65feb161a90e484f2984215f83e | |
| with: | |
| token: ${{ secrets.BOT_TOKEN }} | |
| commit-message: "chore: refresh notebooks" | |
| branch: "refresh-notebooks-for-${{ github.ref_name }}" | |
| base: "${{ github.ref_name }}" | |
| title: "Refresh notebooks for ${{ github.ref_name }}" | |
| body: ${{ env.PR_BODY }} | |
| add-paths: | | |
| docs/**/*.ipynb | |
| stop-runner-linux: | |
| name: Stop EC2 runner | |
| needs: [refresh-notebooks, start-runner-linux] | |
| runs-on: ubuntu-20.04 | |
| if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Stop EC2 runner python 38 | |
| uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e | |
| if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }} | |
| with: | |
| github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} | |
| label: ${{ needs.start-runner-linux.outputs.label-38 }} | |
| ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }} | |
| mode: stop | |
| send-report: | |
| if: ${{ always() }} | |
| needs: | |
| [ | |
| start-runner-linux, | |
| refresh-notebooks, | |
| stop-runner-linux, | |
| ] | |
| name: Send Slack notification | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 | |
| - name: Prepare whole job status | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| env: | |
| NEEDS_JSON: ${{ toJSON(needs) }} | |
| run: | | |
| echo "${NEEDS_JSON}" > /tmp/needs_context.json | |
| JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \ | |
| --needs_context_json /tmp/needs_context.json) | |
| echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV" | |
| - name: Slack Notification | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 | |
| env: | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }} | |
| SLACK_MESSAGE: "Full run finished with status ${{ env.JOB_STATUS || 'failure' }} \ | |
| (${{ env.ACTION_RUN_URL }})\n\ | |
| - start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\ | |
| - refresh-notebooks: ${{ needs.refresh-notebooks.result || 'Did not run.' }}\n\n\ | |
| - stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}" | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} |