tests: add athena integration tests (#503)

* tests: add athena integration tests basic configs

* offer athena testing in github action

* add missing duckdb with TODO for later fix

* add secrets and envirpnment variables configuration

* proper profile config

* aws credentials in github action

* no database and schema being set for duckdb for now

* special case for installing dbt-athena-adapter package for athena adapter

.github/actions/setup-local-fal/action.yml

@@ -37,17 +37,28 @@ runs:
     - name: Install dbt
       shell: bash
       run: |
+        package=dbt-${{ inputs.adapter }}
+        # overwrite for specific adapters
+        if [[ '${{ inputs.adapter }}' == 'athena' ]]
+        then
+          package=dbt-athena-adapter
+        fi
+
         if [[ '${{ inputs.dbt }}' == 'latest' ]]
         then
-          pip install dbt-${{ inputs.adapter }}
+          version='*'
+          pip install ${package}
         elif [[ '${{ inputs.dbt }}' =~ ^1.*$ ]]
         then
-          pip install dbt-${{ inputs.adapter }}==${{ inputs.dbt }}
+          version='${{ inputs.dbt }}'
         else
           echo "Invalid dbt version to install"
           exit 2
         fi
 
+        echo "${package}==${version}"
+        pip install ${package}==${version}
+
     - name: List requirements.txt
       shell: sh
       run: pip freeze

.github/workflows/test_profiles.yml

@@ -31,6 +31,7 @@ on:
           - snowflake
           - redshift
           - duckdb
+          - athena
 
       python:
         description: Python version to test with
@@ -73,6 +74,7 @@ jobs:
           EXTRA_OPTIONS = [
             'redshift',
             'duckdb',
+            'athena',
           ]
           OUTPUT = OPTIONS
 
@@ -210,6 +212,15 @@ jobs:
         if: contains(fromJSON('["redshift", "postgres"]'), matrix.profile)
         run: docker-compose up -d
 
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        # TODO: make redshift use a real connection
+        if: contains(fromJSON('["redshift", "athena"]'), matrix.profile)
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
       - name: Setup behave
         working-directory: integration_tests
         run: pip install behave
@@ -241,6 +252,13 @@ jobs:
           SF_SCHEMA: ${{ secrets.SF_SCHEMA }}
           # Duckdb
           DB_PATH: ${{ github.workspace }}/duck.db
+          # Athena
+          ATHENA_S3_STAGING_DIR: ${{ secrets.ATHENA_S3_STAGING_DIR }}
+          ATHENA_REGION: ${{ secrets.ATHENA_REGION }}
+          ATHENA_WORK_GROUP: ${{ secrets.ATHENA_WORK_GROUP }}
+          ATHENA_PROFILE: ${{ secrets.ATHENA_PROFILE }}
+          ATHENA_DATABASE: ${{ secrets.ATHENA_DATABASE }}
+          ATHENA_SCHEMA: ${{ secrets.ATHENA_SCHEMA }}
         run: |
           # Database and schema setup for sources
           if [[ '${{ matrix.profile }}' == "bigquery" ]]
@@ -255,6 +273,17 @@ jobs:
           then
             export DBT_DATABASE="$SF_DATABASE" DBT_SCHEMA="$SF_SCHEMA"
           fi
+          if [[ '${{ matrix.profile }}' == "duckdb" ]]
+          then
+            # TODO: which to use for sources? Example:
+            #   database: "{{ env_var('DBT_DATABASE', 'test') }}"
+            #   schema: "{{ env_var('DBT_SCHEMA', 'dbt_fal') }}"
+            export DBT_DATABASE="" DBT_SCHEMA=""
+          fi
+          if [[ '${{ matrix.profile }}' == "athena" ]]
+          then
+            export DBT_DATABASE="$ATHENA_DATABASE" DBT_SCHEMA="$ATHENA_SCHEMA"
+          fi
 
           if [[ '${{ matrix.profile }}' == "bigquery" ]]
           then

integration_tests/features/steps/fal_steps.py

@@ -400,7 +400,14 @@ def _get_fal_results_file_name(context):
 
 def _set_profiles_dir(context) -> Path:
     # TODO: Ideally this needs to change in just one place
-    available_profiles = ["postgres", "bigquery", "redshift", "snowflake", "duckdb"]
+    available_profiles = [
+        "postgres",
+        "bigquery",
+        "redshift",
+        "snowflake",
+        "duckdb",
+        "athena",
+    ]
     if "profile" in context.config.userdata:
         profile = context.config.userdata["profile"]
         if profile not in available_profiles:

integration_tests/profiles/athena/profiles.yml

@@ -0,0 +1,15 @@
+config:
+  send_anonymous_usage_stats: False
+
+fal_test:
+  target: integration_tests
+  outputs:
+    integration_tests:
+      type: athena
+      s3_staging_dir: "{{ env_var('ATHENA_S3_STAGING_DIR') }}"
+      region_name: us-east-1
+      database: "{{ env_var('ATHENA_DATABASE') }}"
+      schema: "{{ env_var('ATHENA_SCHEMA') }}"
+      num_retries: 0
+      # work_group: "{{ env_var('ATHENA_WORK_GROUP') }}"
+      # aws_profile_name: "{{ env_var('ATHENA_PROFILE') }}"