Refactored the S3 inactive to new approach #1295
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: PR | |
| on: | |
| pull_request_target: | |
| types: [labeled, synchronize] | |
| branches: [ main ] | |
| jobs: | |
| approve: # First step | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Approve | |
| run: echo For security reasons, all pull requests need to be approved first before running any automated CI. | |
| unittest: | |
| name: unittest | |
| runs-on: ubuntu-latest | |
| needs: [approve] | |
| strategy: | |
| matrix: | |
| python-version: [ '3.8', '3.9', '3.10' ] | |
| # minimize potential vulnerabilities | |
| if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| # ldap requirements | |
| sudo apt update -y | |
| sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y | |
| python -m pip install --upgrade pip | |
| pip install flake8 pytest pytest-cov | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Configure AWS credentials for pytest | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.REGION }} | |
| - name: 📃 Unittest tests with pytest | |
| env: | |
| BUCKET: ${{ secrets.BUCKET }} | |
| REGION: ${{ secrets.REGION }} | |
| run: | | |
| python -m pytest -v tests/unittest | |
| terraform_apply: | |
| name: terraform_apply | |
| needs: [unittest, approve] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| INSTANCE_ID: ${{ steps.terraform_instance_id.outputs.INSTANCE_ID }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Configure AWS credentials for pytest | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.REGION }} | |
| - name: Install terraform and terragrunt | |
| run: | | |
| # Install Terrafrom | |
| curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - | |
| sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | |
| sudo apt-get -y update && sudo apt-get install -y terraform | |
| pip3 install jinja2 | |
| # install terragrunt | |
| wget https://github.com/gruntwork-io/terragrunt/releases/download/v0.38.6/terragrunt_linux_amd64 | |
| mv terragrunt_linux_amd64 terragrunt | |
| mv terragrunt /usr/local/bin/terragrunt | |
| chmod 775 /usr/local/bin/terragrunt | |
| - name: Create instance | |
| id: terraform_instance_id | |
| env: | |
| IMAGE_ID: ${{ secrets.IMAGE_ID }} | |
| INSTANCE_TYPE: ${{ secrets.INSTANCE_TYPE }} | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| ROLE_NAME: ${{ secrets.ROLE_NAME }} | |
| REGION_NAME: ${{ secrets.TERRAFORM_REGION }} | |
| TAG_NAME: ${{ secrets.TAG_NAME }} | |
| run: | | |
| cd terraform/aws_instance | |
| # terrafrom apply | |
| terragrunt apply -auto-approve 1> /dev/null | |
| echo "INSTANCE_ID=$(terragrunt output -raw instance_id)" >> "$GITHUB_OUTPUT" | |
| - name: Cache the Terraform State File | |
| uses: actions/cache@v3 | |
| with: | |
| path: terraform/aws_instance | |
| key: terraform-state-${{ steps.terraform_instance_id.outputs.INSTANCE_ID }} | |
| integration: | |
| name: integration | |
| needs: [ unittest, terraform_apply, approve ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| python-version: [ '3.8', '3.9', '3.10' ] | |
| services: | |
| elasticsearch: | |
| image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 | |
| env: | |
| discovery.type: single-node | |
| options: >- | |
| --health-cmd "curl http://localhost:9200/_cluster/health" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| ports: | |
| # <port on host>:<port on container> | |
| - 9200:9200 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| # ldap requirements | |
| sudo apt update -y | |
| sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y | |
| python -m pip install --upgrade pip | |
| pip install flake8 pytest pytest-cov | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Configure AWS credentials for pytest | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.REGION }} | |
| - name: Set GCP credentials for pytest | |
| env: | |
| GOOGLE_APPLICATION_CREDENTIALS_CONTENTS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} | |
| RUNNER_PATH: ${{ secrets.RUNNER_PATH }} | |
| run: | | |
| echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENTS" > "$RUNNER_PATH/gcp_service.json" | |
| echo "GOOGLE_APPLICATION_CREDENTIALS=$RUNNER_PATH/gcp_service.json" >> "$GITHUB_ENV" | |
| - name: 📃 Integration tests with pytest | |
| env: | |
| BUCKET: ${{ secrets.BUCKET }} | |
| REGION: ${{ secrets.REGION }} | |
| ELASTICSEARCH: 'localhost' | |
| ELASTICSEARCH_PORT: '9200' | |
| INSTANCE_ID: ${{ needs.terraform_apply.outputs.INSTANCE_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| AZURE_ACCOUNT_ID: ${{ secrets.AZURE_ACCOUNT_ID }} | |
| GCP_DATABASE_NAME: ${{ secrets.GCP_DATABASE_NAME }} | |
| GCP_DATABASE_TABLE_NAME: ${{ secrets.GCP_DATABASE_TABLE_NAME }} | |
| run: | | |
| python -m pytest -v tests/integration | |
| terraform_destroy: | |
| name: terraform_destroy | |
| needs: [unittest, terraform_apply, integration, approve] | |
| if: success() || failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Configure AWS credentials for pytest | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.REGION }} | |
| - name: Get Cache of the Terraform State File | |
| uses: actions/cache@v3 | |
| with: | |
| path: terraform/aws_instance | |
| key: terraform-state-${{ needs.terraform_apply.outputs.INSTANCE_ID }} | |
| - name: Install terraform and terragrunt | |
| run: | | |
| # Install Terrafrom | |
| curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - | |
| sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | |
| sudo apt-get -y update && sudo apt-get install -y terraform | |
| pip3 install jinja2 | |
| # install terragrunt | |
| wget https://github.com/gruntwork-io/terragrunt/releases/download/v0.38.6/terragrunt_linux_amd64 | |
| mv terragrunt_linux_amd64 terragrunt | |
| mv terragrunt /usr/local/bin/terragrunt | |
| chmod 775 /usr/local/bin/terragrunt | |
| - name: Destroy instance | |
| env: | |
| ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} | |
| ROLE_NAME: ${{ secrets.ROLE_NAME }} | |
| REGION_NAME: ${{ secrets.TERRAFORM_REGION }} | |
| run: | | |
| cd terraform/aws_instance | |
| # terraform destroy/ | |
| terragrunt destroy -auto-approve 1> /dev/null | |
| e2e: | |
| name: e2e | |
| needs: [ unittest, terraform_apply, integration, approve ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: [ '3.8', '3.9', '3.10' ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| # ldap requirements | |
| sudo apt update -y | |
| sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y | |
| python -m pip install --upgrade pip | |
| pip install flake8 pytest pytest-cov | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Configure AWS credentials for pytest | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.REGION }} | |
| - name: 📃 E2E test | |
| env: | |
| AWS_DEFAULT_REGION: ${{ secrets.REGION }} | |
| policy: ${{ secrets.POLICY }} | |
| policy_output: ${{ secrets.POLICY_OUTPUT }} | |
| RUNNER_PATH: ${{ secrets.RUNNER_PATH }} | |
| run: | | |
| PYTHONPATH=. python cloud_governance/main/main.py 1> /dev/null 2> $RUNNER_PATH/err.log | |
| if [ -s "$RUNNER_PATH/err.log" ]; then echo "Error E2E raised"; fi |