etl.yml

name: Import scraped data to configured database

on:
  workflow_dispatch:
  repository_dispatch:
    types: [nightly-scrape-done]

jobs:
  import_filings:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
        with:
          ref: "deploy"
      - name: Import candidate, PAC, and filing data
        run: |
          touch .env
          docker compose -f docker-compose.etl.yml run --rm \
            -e AWS_STORAGE_BUCKET_NAME=${{ secrets.AWS_STORAGE_BUCKET_NAME }} \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -e DATABASE_URL=${{ secrets.DATABASE_URL }} \
            app make import/candidates import/pacs import/candidate_filings import/pac_filings

  import_transactions:
    runs-on: ubuntu-latest
    needs: import_filings
    strategy:
      matrix:
        transaction_type: [CON, EXP]
        year: [2023, 2024]
        quarter: [1, 2, 3, 4]

    steps:
      - uses: actions/checkout@v3
        with:
          ref: "deploy"
      - name: Import transaction data
        run: |
          touch .env
          docker compose -f docker-compose.etl.yml run --rm \
            -e AWS_STORAGE_BUCKET_NAME=${{ secrets.AWS_STORAGE_BUCKET_NAME }} \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -e DATABASE_URL=${{ secrets.DATABASE_URL }} \
            app make import/${{ matrix.transaction_type }}_${{ matrix.quarter }}_${{ matrix.year }}