Add nightly scrape action #8
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly case scrape | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| schedule: | |
| - cron: '15 4 * * *' | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set current date as env variable | |
| run: echo "BEGIN_COURTS_RUN=$(date +'%s')" >> $GITHUB_ENV | |
| - uses: actions/checkout@v3 | |
| - name: upgrade sqlite3 | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install sqlite3 | |
| - name: Install requirements | |
| run: | | |
| pip install -U pyopenssl cryptography | |
| pip install -r requirements.txt | |
| - name: Download latest database zip | |
| uses: robinraju/[email protected] | |
| with: | |
| latest: true | |
| tag: "nightly" | |
| fileName: "*.db.zip" | |
| - name: Decrypt database | |
| run: | | |
| unzip -P '${{ secrets.CASE_DB_PW }}' cases.db.zip | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| - name: Run scrape | |
| env: | |
| SCRAPY_SETTINGS_MODULE: "courtscraper.settings" | |
| run: | | |
| echo $BEGIN_COURTS_RUN | |
| echo PYTHONPATH="courtscraper" >> $GITHUB_ENV | |
| make get_new_records | |
| - name: Setup database for upload | |
| run: | | |
| zip -p '${{ secrets.CASE_DB_PW }}' cases.db.zip cases.db | |
| - name: Upload new release | |
| uses: WebFreak001/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: https://uploads.github.com/repos/datamade/court-scrapers/releases/131985702/assets{?name,label} | |
| release_id: 131985702 | |
| asset_path: ./cases.db.zip | |
| asset_name: cases.db.zip | |
| asset_content_type: application/zip # required by GitHub API | |
| max_releases: 7 | |
| - name: keepalive | |
| uses: gautamkrishnar/keepalive-workflow@v1 | |
| deploy: | |
| name: Deploy to Heroku | |
| needs: scrape | |
| runs-on: ubuntu-latest | |
| env: | |
| HEROKU_ORGANIZATION: ${{ secrets.HEROKU_ORG }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install requirements | |
| run: pip install -r requirements.txt | |
| - name: Download latest database zip | |
| uses: robinraju/[email protected] | |
| with: | |
| latest: true | |
| tag: "nightly" | |
| fileName: "*.db.zip" | |
| - name: Decrypt database | |
| run: | | |
| unzip -P '${{ secrets.CASE_DB_PW }}' cases.db.zip | |
| - name: Install heroku-builds plugin | |
| run: | | |
| heroku plugins:install heroku-builds | |
| - name: Login to Heroku CLI | |
| uses: akhileshns/[email protected] | |
| with: | |
| heroku_api_key: ${{ secrets.HEROKU_API_KEY }} | |
| heroku_app_name: "" | |
| heroku_email: ${{ secrets.HEROKU_EMAIL }} | |
| justlogin: true | |
| - name: Install Datasette plugins | |
| run: | | |
| datasette install datasette-auth-passwords datasette-auth-tokens | |
| - name: Get hashed Datasette password | |
| run: | | |
| # Store hash as an environment variable | |
| hash=$(echo '${{ secrets.DATASETTE_INSTANCE_PW }}' \ | |
| | datasette hash-password --no-confirm); \ | |
| echo "hash=$hash" >> $GITHUB_ENV | |
| - name: Deploy Datasette instance to Heroku | |
| run: | | |
| datasette publish heroku cases.db \ | |
| -n court-scraper \ | |
| -m metadata.json \ | |
| --install datasette-auth-passwords \ | |
| --plugin-secret datasette-auth-passwords root_password_hash '${{ env.hash }}' |