Skip to content

scrape

scrape #123

Workflow file for this run

name: scrape
on:
workflow_dispatch:
schedule:
- cron: '5 8 * * 0'
jobs:
scrape:
runs-on: ubuntu-latest
steps:
- name: Check out this repo
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Installed package list
run: apt list --installed
- name: Remove Chrome
run: sudo apt purge google-chrome-stable
- name: Remove default Chromium
run: sudo apt purge chromium-browser
- name: Install a new Chromium
run: sudo apt install -y chromium-browser
- name: Install all necessary packages
run: pip install requests beautifulsoup4 pandas tqdm urllib3
- name: Run the scraping script
run: python sc_scraper.py
- name: Commit and push if content changed
run: |-
git config user.name "Automated"
git config user.email "[email protected]"
git add -A
timestamp=$(date -u)
git commit -m "Latest data: ${timestamp}" || exit 0
git push