make the sausage #22197
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This is a basic workflow to help you get started with Actions | |
| name: make the sausage | |
| concurrency: | |
| group: ${{ github.ref_name }} | |
| # Controls when the workflow will run | |
| on: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| timeout: | |
| description: How long to run the scraper for. | |
| required: true | |
| default: 45m | |
| type: string | |
| schedule: | |
| - cron: '12 0,2,4,6,8,10,12,14,16,18,20,22 * * *' | |
| # A workflow run is made up of one or more jobs that can run sequentially or in parallel | |
| jobs: | |
| # This workflow contains a single job called "build" | |
| search-and-index: | |
| # The type of runner that the job will run on | |
| runs-on: ubuntu-latest | |
| # Steps represent a sequence of tasks that will be executed as part of the job | |
| steps: | |
| - name: Inject slug/short variables | |
| uses: rlespinasse/github-slug-action@v4 | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: Install poetry | |
| run: pipx install poetry==1.4.0 | |
| - name: install pypy | |
| id: setup-pypy | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 'pypy-3.8' | |
| cache: 'poetry' | |
| - name: install deps | |
| run: poetry install --no-interaction | |
| - name: install b2 | |
| run: pipx install b2==3.11.0 | |
| - name: configure b2 | |
| run: b2 authorize-account $B2_KEY_ID $B2_APPLICATION_KEY | |
| env: | |
| B2_APPLICATION_KEY: ${{ secrets.B2_APPLICATION_KEY }} | |
| B2_KEY_ID: ${{ secrets.B2_KEY_ID }} | |
| - name: download current db if it exists... | |
| run: b2 download-file-by-name rdsqlite backups/orchard-${{ env.GITHUB_REF_SLUG }}.db orchard.db | |
| working-directory: ./orchard/scan | |
| continue-on-error: false | |
| - name: Print out the sha1sum of the current db | |
| run: sha1sum orchard.db | |
| working-directory: ./orchard/scan | |
| continue-on-error: true | |
| - name: scan | |
| run: timeout ${{ github.event.inputs.timeout || '45m' }} poetry run python scan.py orchard.db sources.yml | |
| working-directory: ./orchard/scan | |
| continue-on-error: true | |
| env: | |
| B2_ACCESS_KEY: ${{ secrets.B2_APPLICATION_KEY }} | |
| KEY_ID: ${{ secrets.B2_KEY_ID }} | |
| BOT_TOKEN: ${{ secrets.BOT_TOKEN }} | |
| - name: upload to b2 | |
| run: b2 upload-file rdsqlite orchard.db backups/orchard-${{ env.GITHUB_REF_SLUG }}.db | |
| working-directory: ./orchard/scan | |
| prepare-and-upload: | |
| needs: search-and-index | |
| runs-on: ubuntu-latest | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Inject slug/short variables | |
| uses: rlespinasse/[email protected] | |
| - name: install litestream | |
| run: wget https://github.com/benbjohnson/litestream/releases/download/v0.3.9/litestream-v0.3.9-linux-amd64.deb && sudo dpkg -i litestream-v0.3.9-linux-amd64.deb | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: Install poetry | |
| run: pipx install poetry==1.4.0 | |
| - name: install pypy | |
| id: setup-pypy | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 'pypy-3.8' | |
| cache: 'poetry' | |
| - name: install deps | |
| run: poetry install --no-interaction | |
| - name: install b2 | |
| run: pipx install b2 | |
| - name: configure b2 | |
| run: b2 authorize-account $B2_KEY_ID $B2_APPLICATION_KEY | |
| env: | |
| B2_APPLICATION_KEY: ${{ secrets.B2_APPLICATION_KEY }} | |
| B2_KEY_ID: ${{ secrets.B2_KEY_ID }} | |
| - name: print b2 account info | |
| run: b2 get-account-info | |
| - name: download DB from previous job | |
| run: b2 download-file-by-name rdsqlite backups/orchard-${{ env.GITHUB_REF_SLUG }}.db orchard.db | |
| working-directory: ./orchard/package | |
| - name: get the status db from litestream | |
| run: litestream restore -v -config orchard/bot/litestream.yml ./orchard/package/status.db | |
| env: | |
| KEY_ID: ${{ secrets.B2_KEY_ID }} | |
| B2_ACCESS_KEY: ${{ secrets.B2_APPLICATION_KEY }} | |
| LITESTREAM_PATH: status-main | |
| LITESTREAM_DB: ./orchard/package/status.db | |
| - name: print out hashes of the aforementioned dbs | |
| run: sha1sum *.db | |
| working-directory: ./orchard/package | |
| - name: package... | |
| run: poetry run python package.py orchard.db status.db | |
| working-directory: ./orchard/package | |
| - name: print out hashes of the resulting jsonl file | |
| run: sha1sum orchard.jsonl | |
| working-directory: ./orchard/package | |
| - name: install flyctl | |
| uses: superfly/flyctl-actions/setup-flyctl@master | |
| - name: fly me to the moon | |
| run: flyctl deploy --remote-only | |
| env: | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| working-directory: ./orchard/package |