PR.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: PR

on:
  pull_request_target:
    types: [labeled, synchronize]
    branches: [ main ]

jobs:
  approve: # First step
    runs-on: ubuntu-latest
    steps:
      - name: Approve
        run: echo For security reasons, all pull requests need to be approved first before running any automated CI.
  unittest:
    name: unittest
    runs-on: ubuntu-latest
    needs: [approve]
    strategy:
      matrix:
        python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
    # minimize potential vulnerabilities
    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
    steps:
    - uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        # ldap requirements
        sudo apt update -y
        sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
        python -m pip install --upgrade pip
        pip install flake8 pytest pytest-cov
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
        if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
    - name: Configure AWS credentials for pytest
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }}
        aws-region: ${{ secrets.REGION }}
    - name: 📃 Unittest tests with pytest
      env:
        BUCKET: ${{ secrets.BUCKET }}
        REGION: ${{ secrets.REGION }}
      run: |
        python -m pytest -v tests/unittest

  terraform_apply:
    name: terraform_apply
    needs: [unittest, approve]
    runs-on: ubuntu-latest
    outputs:
      INSTANCE_ID: ${{ steps.terraform_instance_id.outputs.INSTANCE_ID }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Configure AWS credentials for pytest
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.REGION }}
      - name: Install terraform and terragrunt
        run: |
          # Install Terrafrom
          curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
          sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
          sudo apt-get -y update && sudo apt-get install -y terraform
          pip3 install jinja2
          # install terragrunt
          wget https://github.com/gruntwork-io/terragrunt/releases/download/v0.38.6/terragrunt_linux_amd64
          mv terragrunt_linux_amd64 terragrunt
          mv terragrunt /usr/local/bin/terragrunt
          chmod 775 /usr/local/bin/terragrunt
      - name: Create instance
        id: terraform_instance_id
        env:
          IMAGE_ID: ${{ secrets.IMAGE_ID }}
          INSTANCE_TYPE: ${{ secrets.INSTANCE_TYPE }}
          ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }}
          ROLE_NAME: ${{ secrets.ROLE_NAME }}
          REGION_NAME: ${{ secrets.TERRAFORM_REGION }}
          TAG_NAME: ${{ secrets.TAG_NAME }}
        run: |
          cd terraform/aws_instance
          # terrafrom apply
          terragrunt apply -auto-approve 1> /dev/null
          echo "INSTANCE_ID=$(terragrunt output -raw instance_id)" >> "$GITHUB_OUTPUT"
      - name: Cache the Terraform State File
        uses: actions/cache@v4
        with:
          path: terraform/aws_instance
          key: terraform-state-${{ steps.terraform_instance_id.outputs.INSTANCE_ID }}

  integration:
    name: integration
    needs: [ unittest, terraform_apply, approve ]
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 1
      matrix:
        python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
    services:
      elasticsearch:
        image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
        env:
          discovery.type: single-node
        options: >-
          --health-cmd "curl http://localhost:9200/_cluster/health"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
        ports:
          # <port on host>:<port on container>
          - 9200:9200
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          # ldap requirements
          sudo apt update -y
          sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
          python -m pip install --upgrade pip
          pip install flake8 pytest pytest-cov
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
      - name: Configure AWS credentials for pytest
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.REGION }}
      - name: Set GCP credentials for pytest
        env:
          GOOGLE_APPLICATION_CREDENTIALS_CONTENTS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
          RUNNER_PATH: ${{ secrets.RUNNER_PATH }}
        run: |
          echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENTS" > "$RUNNER_PATH/gcp_service.json"
          echo "GOOGLE_APPLICATION_CREDENTIALS=$RUNNER_PATH/gcp_service.json" >> "$GITHUB_ENV"
      - name: 📃 Integration tests with pytest
        env:
          BUCKET: ${{ secrets.BUCKET }}
          REGION: ${{ secrets.REGION }}
          ELASTICSEARCH: 'localhost'
          ELASTICSEARCH_PORT: '9200'
          INSTANCE_ID: ${{ needs.terraform_apply.outputs.INSTANCE_ID }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_ACCOUNT_ID: ${{ secrets.AZURE_ACCOUNT_ID }}
          GCP_DATABASE_NAME: ${{ secrets.GCP_DATABASE_NAME }}
          GCP_DATABASE_TABLE_NAME: ${{ secrets.GCP_DATABASE_TABLE_NAME }}
        run: |
          python -m pytest -v tests/integration

  terraform_destroy:
    name: terraform_destroy
    needs: [unittest, terraform_apply, integration, approve]
    if: success() || failure()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Configure AWS credentials for pytest
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.REGION }}
      - name: Get Cache of the Terraform State File
        uses: actions/cache@v4
        with:
          path: terraform/aws_instance
          key: terraform-state-${{ needs.terraform_apply.outputs.INSTANCE_ID }}
      - name: Install terraform and terragrunt
        run: |
          # Install Terrafrom
          curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
          sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
          sudo apt-get -y update && sudo apt-get install -y terraform
          pip3 install jinja2
          # install terragrunt
          wget https://github.com/gruntwork-io/terragrunt/releases/download/v0.38.6/terragrunt_linux_amd64
          mv terragrunt_linux_amd64 terragrunt
          mv terragrunt /usr/local/bin/terragrunt
          chmod 775 /usr/local/bin/terragrunt
      - name: Destroy instance
        env:
          ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }}
          ROLE_NAME: ${{ secrets.ROLE_NAME }}
          REGION_NAME: ${{ secrets.TERRAFORM_REGION }}
        run: |
          cd terraform/aws_instance
          # terraform destroy/
          terragrunt destroy -auto-approve 1> /dev/null

  e2e:
    name: e2e
    needs: [ unittest, terraform_apply, integration, approve  ]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          # ldap requirements
          sudo apt update -y
          sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
          python -m pip install --upgrade pip
          pip install flake8 pytest pytest-cov
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
      - name: Configure AWS credentials for pytest
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.REGION }}
      - name: 📃 E2E test
        env:
          AWS_DEFAULT_REGION: ${{ secrets.REGION }}
          policy: ${{ secrets.POLICY }}
          policy_output: ${{ secrets.POLICY_OUTPUT }}
          RUNNER_PATH: ${{ secrets.RUNNER_PATH }}
        run: |
          PYTHONPATH=. python cloud_governance/main/main.py 1> /dev/null 2> $RUNNER_PATH/err.log
          if [ -s "$RUNNER_PATH/err.log" ]; then echo "Error E2E raised"; fi