interval-crawler-task.yml

name: Interval Crawler Task

on:
  schedule:
    - cron: '35 8 * 7-9,12 *'
    - cron: '35 * * 1-6,10-11 *'
  pull_request:
    branches: ['main']
  workflow_dispatch:

concurrency:
  group: interval-crawler
  cancel-in-progress: true

jobs:
  crawl-and-push:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
        with:
          token: ${{ secrets.GH_PAT }}

      - name: Pull the latest data branch
        run: |
          git config user.name ZKLlab-bot
          git config user.email zkllab-bot@mail.zkllab.com
          git subtree add --prefix=interval-crawler-task-result origin data

      - name: Duplicate data folder
        run: |
          cp -r interval-crawler-task-result data

      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9

      - name: Install OpenVPN Client
        run: |
          sudo apt-get update && sudo apt-get -y install openvpn

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pipenv
          pipenv install

      - name: Connect to OpenVPN
        run: |
          echo "${{ secrets.SHU_USERNAME }}" > /tmp/ovpn-auth.txt
          echo "${{ secrets.SHU_PASSWORD }}" >> /tmp/ovpn-auth.txt
          sudo openvpn --config ./shu-student.ovpn --auth-user-pass /tmp/ovpn-auth.txt --daemon

      - name: Wait for OpenVPN connection
        timeout-minutes: 2
        run: |
          until ping -c1 xk.autoisp.shu.edu.cn; do sleep 2; done

      - name: Setup Node.js environment
        uses: actions/setup-node@v4.0.2
        with:
          node-version: 18.x
          cache: 'yarn'

      - name: Install dependencies
        run: yarn install

      - name: Fetch data
        env:
          SHUSTUID: ${{ secrets.SHU_USERNAME }}
          SHUSTUPWD: ${{ secrets.SHU_PASSWORD }}
          OUTPUTDIR: interval-crawler-task-result
        run: yarn start

      - name: Check Results
        run: |
          tree -I node_modules

      - name: Shutdown OpenVPN
        run: |
          sudo killall openvpn

      - name: Compare, push, and create a pull request if necessary
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          pipenv run python post_crawler.py