cd.yml

name: Data Engineering - Deploy to AWS Glue

on:
  push:
    branches:
      - main

jobs:
    deploy:
        runs-on: ubuntu-latest
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
          S3_SCRIPTS_PATH: ${{ secrets.S3_SCRIPTS_PATH }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
        steps:
            - uses: actions/checkout@v2

            - name: Set up Python
              uses: actions/setup-python@v2
              with:
                python-version: 3.9

            - run: |
                pip install setuptools wheel
                python3 setup.py bdist_wheel

            # Step 1: Copy script to S3 bucket
            - name: Copy script to S3 bucket
              uses: jakejarvis/s3-sync-action@v0.5.0
              with:
                args: --follow-symlinks
              env:
                SOURCE_DIR: "dist"
                DEST_DIR: $S3_SCRIPTS_PATH
                AWS_S3_BUCKET: $S3_BUCKET_NAME