Crawling #28272

	name: Crawling
	on:
	push:
	branches:
	- main
	schedule:
	- cron: "/30 * * *"
	concurrency:
	group: crawling

	jobs:
	crawling:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v2
	with:
	persist-credentials: false

	- name: Checkout data
	uses: actions/checkout@v2
	with:
	persist-credentials: false
	ref: gh-pages
	path: ./data

	- name: Install
	run: yarn install --frozen-lockfile

	- name: Pip
	uses: actions/setup-python@v4
	with:
	python-version: '3.9'
	cache: 'pip' # caching pip dependencies

	- name: Pip Install
	run: pip install -r requirements.txt

	- name: Crawling
	run: yarn start
	env:
	LOG_FORMAT: json
	NUM_TERMS: 2
	ALWAYS_SCRAPE_CURRENT_TERM: 1
	DETAILS_CONCURRENCY: 256
	DATA_FOLDER: ./data
	NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/intermediate.pem

	- name: Revision
	run: python ./src/Revise.py

	- name: Upload
	uses: JamesIves/github-pages-deploy-action@releases/v4
	with:
	token: ${{ secrets.CRAWLER_DEPLOY_PERSONAL_ACCESS_TOKEN }}
	branch: gh-pages
	folder: ./data
	clean: true
	single-commit: true
	git-config-name: gt-scheduler-bot
	git-config-email: [email protected]

Provide feedback