crawling.yml

name: Crawling
on:
  push:
    branches:
      - main
  schedule:
    - cron: "*/30 * * * *"

jobs:
  crawling:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          persist-credentials: false

      - name: Checkout data
        uses: actions/checkout@v2
        with:
          persist-credentials: false
          ref: gh-pages
          path: ./data

      - name: Install
        run: yarn install --frozen-lockfile
        
      - name: Pip
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
          cache: 'pip' # caching pip dependencies

      - name: Pip Install
        run: pip install -r requirements.txt

      - name: Crawling
        run: yarn start
        env:
          LOG_FORMAT: json
          NUM_TERMS: 2
          ALWAYS_SCRAPE_CURRENT_TERM: 1
          DETAILS_CONCURRENCY: 256
          DATA_FOLDER: ./data
          
      - name: Revision
        run: python ./src/Revise.py

      - name: Upload
        uses: JamesIves/github-pages-deploy-action@releases/v4
        with:
          token: ${{ secrets.CRAWLER_DEPLOY_PERSONAL_ACCESS_TOKEN }}
          branch: gh-pages
          folder: ./data
          clean: true
          single-commit: true
          git-config-name: gt-scheduler-bot
          git-config-email: 89671168+gt-scheduler-bot@users.noreply.github.com