Skip to content

Commit

Permalink
🐛 Fix GKE cloud test
Browse files Browse the repository at this point in the history
`GitHubSecurityLab/actions-permissions/monitor` added a MITM proxy to intercept calls.
This proxy signed the GKE API cert which lead to errors:
```
Unable to connect to the server: tls: failed to verify certificate: x509: certificate signed by unknown authority
```

Signed-off-by: Christian Zunker <[email protected]>
  • Loading branch information
czunker committed Nov 9, 2023
1 parent 6756963 commit c072b00
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 102 deletions.
2 changes: 2 additions & 0 deletions .github/terraform/gke/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ resource "google_container_cluster" "cluster" {

remove_default_node_pool = true
initial_node_count = 1
deletion_protection = false
}

resource "google_container_node_pool" "node_pool" {
name = "mondoo-operator-pool-${random_string.suffix.result}"
location = "us-central1-a"
project = var.project_id
cluster = google_container_cluster.cluster.id
node_count = 1

Expand Down
2 changes: 1 addition & 1 deletion .github/terraform/gke/outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ resource "local_file" "kubeconfig" {
depends_on = [google_container_cluster.cluster]
content = module.gke_auth.kubeconfig_raw
filename = "kubeconfig"
}
}
191 changes: 94 additions & 97 deletions .github/workflows/cloud-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,93 +38,90 @@ env:
CNSPEC_IMAGE_TAG: ${{ github.event.inputs.cnspecImageTag || 'edge-latest-rootless' }}

jobs:
aks-integration-test:
runs-on: ubuntu-latest
name: AKS integration tests

env:
ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }}
# aks-integration-test:
# runs-on: ubuntu-latest
# name: AKS integration tests

# env:
# ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
# ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
# ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
# ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
# KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/aks/kubeconfig') }}

strategy:
fail-fast: false
matrix:
k8s-version: ["1.25", "1.26", "1.27"]
# strategy:
# fail-fast: false
# matrix:
# k8s-version: ["1.25", "1.26", "1.27"]

steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
# steps:
# - uses: actions/checkout@v4
# with:
# fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile

- name: Import environment variables from file
run: cat ".github/env" >> $GITHUB_ENV
# - name: Import environment variables from file
# run: cat ".github/env" >> $GITHUB_ENV

- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
# - name: Setup Terraform
# uses: hashicorp/setup-terraform@v3

- name: Terraform init
run: terraform init
working-directory: .github/terraform/aks
# - name: Terraform init
# run: terraform init
# working-directory: .github/terraform/aks

- name: Terraform plan
run: terraform plan -out aks-${{ matrix.k8s-version }}.json
env:
TF_VAR_k8s_version: ${{ matrix.k8s-version }}
working-directory: .github/terraform/aks

- name: Terraform apply
run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json
env:
TF_VAR_k8s_version: ${{ matrix.k8s-version }}
working-directory: .github/terraform/aks

- uses: actions/setup-go@v4
with:
go-version: "${{ env.golang-version }}"
cache: true

- name: Get operator version
run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

- name: Wait a bit for the cluster to become more stable
run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

- name: Run integration tests
env:
MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci

- name: Clean up AKS terraform
run: terraform destroy -auto-approve
if: success() || failure()
working-directory: .github/terraform/aks
# - name: Terraform plan
# run: terraform plan -out aks-${{ matrix.k8s-version }}.json
# env:
# TF_VAR_k8s_version: ${{ matrix.k8s-version }}
# working-directory: .github/terraform/aks

# - name: Terraform apply
# run: terraform apply -auto-approve aks-${{ matrix.k8s-version }}.json
# env:
# TF_VAR_k8s_version: ${{ matrix.k8s-version }}
# working-directory: .github/terraform/aks

# - uses: actions/setup-go@v4
# with:
# go-version: "${{ env.golang-version }}"
# cache: true

# - name: Get operator version
# run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

# - name: Wait a bit for the cluster to become more stable
# run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s

# - name: Run integration tests
# env:
# MONDOO_API_TOKEN: ${{ secrets.MONDOO_TEST_ORG_TOKEN }}
# MONDOO_ORG_MRN: //captain.api.mondoo.app/organizations/serene-lovelace-854342
# MONDOO_GQL_ENDPOINT: https://api.edge.mondoo.com/query
# run: VERSION=${{ env.OPERATOR_VERSION }} K8S_DISTRO=aks make test/integration/ci

# - name: Clean up AKS terraform
# run: terraform destroy -auto-approve
# if: success() || failure()
# working-directory: .github/terraform/aks

- run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml
if: success() || failure()

- name: Upload cloud test results
uses: actions/upload-artifact@v3 # upload test results
if: success() || failure() # run this step even if previous step failed
with: # upload a combined archive with unit and integration test results
name: cloud-test-results
path: |
integration-tests-aks-${{ matrix.k8s-version }}.xml
.github/terraform/aks/aks-${{ matrix.k8s-version }}.json
- name: Upload test logs artifact
uses: actions/upload-artifact@v3
if: failure()
with:
name: test-logs-aks-${{ matrix.k8s-version }}
path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/
# - run: mv integration-tests.xml integration-tests-aks-${{ matrix.k8s-version }}.xml
# if: success() || failure()

# - name: Upload cloud test results
# uses: actions/upload-artifact@v3 # upload test results
# if: success() || failure() # run this step even if previous step failed
# with: # upload a combined archive with unit and integration test results
# name: cloud-test-results
# path: |
# integration-tests-aks-${{ matrix.k8s-version }}.xml
# .github/terraform/aks/aks-${{ matrix.k8s-version }}.json

# - name: Upload test logs artifact
# uses: actions/upload-artifact@v3
# if: failure()
# with:
# name: test-logs-aks-${{ matrix.k8s-version }}
# path: /home/runner/work/mondoo-operator/mondoo-operator/tests/integration/_output/

eks-integration-test:
runs-on: ubuntu-latest
Expand All @@ -133,7 +130,7 @@ jobs:
strategy:
fail-fast: false
matrix:
k8s-version: ["1.24", "1.25", "1.26", "1.27", "1.28"]
k8s-version: ["1.25"] #, "1.25", "1.26", "1.27", "1.28"]

env:
TF_VAR_test_name: ${{ github.event.inputs.mondooOperatorImageTag }}
Expand All @@ -142,9 +139,6 @@ jobs:
AWS_REGION: us-east-2

steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
Expand All @@ -155,6 +149,13 @@ jobs:
- name: Setup Terraform
uses: hashicorp/setup-terraform@v3

- name: Where am I?
run: |
wget https://releases.mondoo.com/cnquery/9.6.0/cnquery_9.6.0_linux_amd64.tar.gz
tar -xzf cnquery_9.6.0_linux_amd64.tar.gz
./cnquery run aws -c "aws.account{ id aliases } aws.vpcs.where(region == 'us-east-2'){ id region }"
./cnquery run aws -c "aws.vpcs.where(region == 'eu-central-1'){ id region }"
- run: terraform init
working-directory: .github/terraform/aws

Expand Down Expand Up @@ -217,16 +218,13 @@ jobs:
strategy:
fail-fast: false
matrix:
k8s-version: ["1.25", "1.26", "1.27"]
k8s-version: ["1.25"] #, "1.26", "1.27"]

env:
GOOGLE_APPLICATION_CREDENTIALS: ${{ format('{0}/{1}', github.workspace, 'gcp_sa.json') }}
KUBECONFIG: ${{ format('{0}/{1}', github.workspace, '.github/terraform/gke/kubeconfig') }}

steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
Expand Down Expand Up @@ -260,11 +258,14 @@ jobs:
go-version: "${{ env.golang-version }}"
cache: true

- name: Manually install dep
run: go get github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/[email protected]

- name: Get operator version
run: echo "OPERATOR_VERSION=$(docker run ghcr.io/mondoohq/mondoo-operator:${{ env.MONDOO_OPERATOR_IMAGE_TAG }} version --simple)" >> $GITHUB_ENV

- name: Wait a bit for the cluster to become more stable
run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=60s
run: kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=300s

- name: Run integration tests
env:
Expand Down Expand Up @@ -302,12 +303,10 @@ jobs:
test-report:
name: Report test results
runs-on: ubuntu-latest
needs: [eks-integration-test,aks-integration-test,gke-integration-test]
#needs: [eks-integration-test,aks-integration-test,gke-integration-test]
needs: [gke-integration-test]
if: always()
steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch is needed for "git tag --list" in the Makefile
Expand All @@ -327,13 +326,11 @@ jobs:
discord-notification:
runs-on: ubuntu-latest
name: Send Discord notification
needs: [eks-integration-test,aks-integration-test,gke-integration-test]
#needs: [eks-integration-test,aks-integration-test,gke-integration-test]
needs: [gke-integration-test]
# Run only if the previous job has failed and only if it's running against the main branch
if: ${{ always() && contains(join(needs.*.result, ','), 'fail') && github.ref_name == 'main' }}
steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- uses: sarisia/actions-status-discord@v1
with:
webhook: ${{ secrets.DISCORD_WEBHOOK }}
Expand Down
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ require (
github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 // indirect
github.com/GoogleCloudPlatform/berglas v1.0.3 // indirect
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.9.2 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/OpenPeeDeeP/depguard/v2 v2.1.0 // indirect
Expand Down Expand Up @@ -118,9 +119,9 @@ require (
github.com/go-critic/go-critic v0.9.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonpointer v0.20.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/go-toolsmith/astcast v1.1.0 // indirect
github.com/go-toolsmith/astcopy v1.1.0 // indirect
github.com/go-toolsmith/astequal v1.1.0 // indirect
Expand Down Expand Up @@ -208,7 +209,7 @@ require (
github.com/mbilski/exhaustivestruct v1.2.0 // indirect
github.com/mgechev/revive v1.3.4 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moricho/tparallel v0.3.1 // indirect
github.com/mtibben/percent v0.2.1 // indirect
Expand Down Expand Up @@ -326,7 +327,7 @@ require (
github.com/google/go-containerregistry v0.16.1
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.4.0
github.com/imdario/mergo v0.3.12 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
Expand Down
8 changes: 8 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 h1:3ZBs7LAezy8gh0uECsA6C
github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0/go.mod h1:rZLTje5A9kFBe0pzhpe2TdhRniBF++PRHQuRpR8esVc=
github.com/GoogleCloudPlatform/berglas v1.0.3 h1:NjJYDz13vWct7+joxkBkIZhD6Cmwf5XP5t0jGTvHyJk=
github.com/GoogleCloudPlatform/berglas v1.0.3/go.mod h1:JBsGyi6Z5RwyHXMdEebok6MChukLE+dWXzPor2aeMtw=
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.9.2 h1:7fdp02N9fd8itrSe/p7njaSKAUYJGgxn8ajgZfbFK+I=
github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.9.2/go.mod h1:yyde2qkA+GhCou8exSJwifnJlAcWCNcU1vs911CEOJg=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
Expand Down Expand Up @@ -324,10 +326,12 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA=
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
Expand Down Expand Up @@ -540,6 +544,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
Expand Down Expand Up @@ -675,6 +680,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 h1:drhDO54gdT/a15GBcMRmunZiNcLgPiFIJa23KzmcvcU=
github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770/go.mod h1:SO/iHr6q2EzbqRApt+8/E9wqebTwQn5y+UlB04bxzo0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
Expand Down Expand Up @@ -728,6 +735,7 @@ github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVn
github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
github.com/otiai10/copy v1.11.0 h1:OKBD80J/mLBrwnzXqGtFCzprFSGioo30JcmR4APsNwc=
github.com/otiai10/copy v1.11.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
Expand Down

0 comments on commit c072b00

Please sign in to comment.