Skip to content

Commit

Permalink
Add backend switching job for EventMesh on Push event (#400)
Browse files Browse the repository at this point in the history
* Add backend switching job for EventMesh on Push event

* Apply suggestions from code review

Co-authored-by: Friedrich <[email protected]>

* added comments

* updated comments to use #

---------

Co-authored-by: Friedrich <[email protected]>
  • Loading branch information
mfaizanse and friedrichwilken authored Jan 16, 2024
1 parent e4d3a96 commit 44ff053
Show file tree
Hide file tree
Showing 10 changed files with 831 additions and 2 deletions.
192 changes: 192 additions & 0 deletions .github/workflows/e2e-backend-switching-reuseable.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
name: Backend-switching-tests-gardener (reusable)

on:
workflow_call:
inputs:
eventing-manager-image:
description: Container image of eventing-manger which needs to be tested.
required: true
type: string
kube-version:
description: Kubernetes version for the Gardener cluster.
required: true
type: string
cluster-name-prefix:
description: Prefix to use as prefix for name of Gardener cluster.
required: false
type: string
default: "ghem-"

jobs:
backend-switching:
runs-on: ubuntu-latest
env:
KYMA_STABILITY: "unstable"
KYMA: "./hack/kyma"
MANAGER_IMAGE: ${{ inputs.eventing-manager-image }}

steps:
- uses: actions/checkout@v4

- name: Cache Binaries
id: cache-binaries
uses: actions/cache@v3
with:
path: bin
key: ${{ runner.os }}-bin

- name: Install Kyma CLI
run: |
make kyma
- name: Provision Gardener cluster
env:
GARDENER_CLUSTER_VERSION: ${{ inputs.kube-version }}
CLUSTER_PREFIX: ${{ inputs.cluster-name-prefix }}
GARDENER_REGION: "eu-west-1"
GARDENER_ZONES: "eu-west-1a"
GARDENER_PROJECT_NAME: ${{ vars.GARDENER_PROJECT_NAME }}
GARDENER_PROVIDER_SECRET_NAME: ${{ vars.GARDENER_PROVIDER_SECRET_NAME_AWS }}
MACHINE_TYPE: "c4.xlarge"
SCALER_MIN: "1"
SCALER_MAX: "2"
RETRY_ATTEMPTS: "2"
GARDENER_KYMATUNAS: ${{ secrets.GARDENER_KYMATUNAS }}
run: |
# setup Gardener kubeconfig.
mkdir -p "${HOME}/.gardener"
export GARDENER_KUBECONFIG="${HOME}/.gardener/kubeconfig"
echo ${GARDENER_KYMATUNAS} | base64 --decode > ${GARDENER_KUBECONFIG}
# generate cluster name and export it to Github env for cleanup step to access it.
export CLUSTER_NAME="${CLUSTER_PREFIX}$(openssl rand -hex 2)"
echo "CLUSTER_NAME=${CLUSTER_NAME}" >> $GITHUB_ENV
# provision gardener cluster.
make -C hack/ci/ provision-gardener-cluster
kubectl version
kubectl cluster-info
kubectl get nodes
kubectl get ns
- name: Create kyma-system namespace
run: |
kubectl create ns kyma-system || true
- name: Create EventMesh secret
env:
EVENTMESH_K8S_SECRET: ${{ secrets.EVENTMESH_K8S_SECRET }}
run: |
echo "${EVENTMESH_K8S_SECRET}" | base64 --decode > k8s-em.yaml
kubectl apply -n kyma-system -f k8s-em.yaml
rm k8s-em.yaml
- name: Create IAS application for EventMesh
env:
TEST_EVENTING_AUTH_IAS_URL: ${{ vars.EVENTING_AUTH_IAS_URL }}
TEST_EVENTING_AUTH_IAS_USER: ${{ vars.EVENTING_AUTH_IAS_USER }}
TEST_EVENTING_AUTH_IAS_PASSWORD: ${{ secrets.EVENTING_AUTH_IAS_PASSWORD }}
run: |
export DISPLAY_NAME=${CLUSTER_NAME}
make -C hack/ci/ create-ias-app
- name: Install latest released Istio Module
run: |
make -C hack/ci/ install-istio-module
- name: Install latest released API Gateway Manager
run: |
make -C hack/ci/ install-api-gateway-module
- name: Install latest released NATS Manager
run: |
make -C hack/ci/ install-nats-module
- name: Deploy eventing-manager
run: |
make install
make deploy IMG=$MANAGER_IMAGE
kubectl apply -f config/samples/default.yaml
- name: Wait for Installed modules to be ready
run: |
make -C hack/ci/ wait-istio-cr-ready
make -C hack/ci/ wait-api-gateway-cr-ready
make -C hack/ci/ wait-nats-cr-ready
make -C hack/ci/ wait-eventing-cr-ready-with-backend ACTIVE_BACKEND=NATS
- name: Setup eventing tests
run: |
make e2e-eventing-setup
- name: Test eventing with NATS
run: |
make e2e-eventing
- name: Switch to EventMesh backend
run: |
kubectl apply -f config/samples/default_eventmesh.yaml
make -C hack/ci/ wait-eventing-cr-ready-with-backend ACTIVE_BACKEND=EventMesh
- name: Test eventing with EventMesh
env:
BACKEND_TYPE: "EventMesh"
run: |
# wait for subscriptions to be ready.
make e2e-eventing-setup
# run tests.
make e2e-eventing
- name: Switch back to NATS backend
run: |
kubectl apply -f config/samples/default.yaml
make -C hack/ci/ wait-eventing-cr-ready-with-backend ACTIVE_BACKEND=NATS
- name: Test eventing again with NATS
run: |
# wait for subscriptions to be ready.
make e2e-eventing-setup
# run tests.
make e2e-eventing
- name: Test eventing cleanup
run: |
make e2e-cleanup
- name: On error, fetch NATS CR
if: failure()
run: |
kubectl get nats.operator.kyma-project.io -n kyma-system -o yaml
- name: On error, fetch Eventing CRs
if: failure()
run: |
kubectl get eventing.operator.kyma-project.io -n kyma-system -o yaml
- name: On error, fetch Istio CR
if: failure()
run: |
kubectl get istios.operator.kyma-project.io -n kyma-system -o yaml
- name: On error, fetch API Gateway CR
if: failure()
run: |
kubectl get apigateways.operator.kyma-project.io -n kyma-system -o yaml
- name: Delete IAS application
if: ${{ always() }}
env:
TEST_EVENTING_AUTH_IAS_URL: ${{ vars.EVENTING_AUTH_IAS_URL }}
TEST_EVENTING_AUTH_IAS_USER: ${{ vars.EVENTING_AUTH_IAS_USER }}
TEST_EVENTING_AUTH_IAS_PASSWORD: ${{ secrets.EVENTING_AUTH_IAS_PASSWORD }}
run: |
export IAS_APPLICATION_LOCATION=$(cat ~/.ias_location)
make -C hack/ci/ delete-ias-app
- name: Delete Gardener cluster
if: ${{ always() }}
env:
GARDENER_PROVIDER_SECRET_NAME: "tunas-aws"
GARDENER_PROJECT_NAME: "kymatunas"
WAIT_FOR_DELETE_COMPLETION: "false"
run: |
export GARDENER_KUBECONFIG="${HOME}/.gardener/kubeconfig"
make -C hack/ci/ deprovision-gardener-cluster
58 changes: 56 additions & 2 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,20 @@ name: e2e-without-lifecycle-manager
env:
KYMA_STABILITY: "unstable"
KYMA: "./hack/kyma"
MANAGER_IMAGE: europe-docker.pkg.dev/kyma-project/dev/eventing-manager:PR-${{ github.event.number }}

on:
push:
branches:
- main
- "release-**"
paths-ignore:
- "docs/**"
- "**.md"
- "sec-scanners-config.yaml"
pull_request:
branches:
- main
- "release-*"
- "release-**"
paths-ignore:
- "docs/**"
- "**.md"
Expand All @@ -18,8 +25,14 @@ on:
jobs:
wait-until-build-succeeds:
runs-on: ubuntu-latest
outputs:
image-name: ${{ steps.extract-image.outputs.image_name }}
steps:
- uses: actions/checkout@v4

- name: Wait for the 'pull-eventing-manager-build' job to succeed
id: pull-wait-build
if: github.event_name == 'pull_request'
uses: kyma-project/wait-for-commit-status-action@2b3ffe09af8b6f40e1213d5fb7f91a7bd41ffb20
with:
context: "pull-eventing-manager-build"
Expand All @@ -32,9 +45,38 @@ jobs:
GITHUB_OWNER: "${{ github.repository_owner }}"
GITHUB_REPO: "eventing-manager"

- name: Wait for the 'post-eventing-manager-build' job to succeed
id: push-wait-build
if: github.event_name == 'push'
uses: kyma-project/wait-for-commit-status-action@2b3ffe09af8b6f40e1213d5fb7f91a7bd41ffb20
with:
context: "post-eventing-manager-build"
commit_ref: "${{ github.sha }}"
timeout: 600000 # 10 minutes in milliseconds
# The check interval is kept long otherwise it will exhaust the GitHub rate limit (More info: https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#rate-limiting)
check_interval: 60000 # 1 minute in milliseconds
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
GITHUB_OWNER: "${{ github.repository_owner }}"
GITHUB_REPO: "eventing-manager"

- name: Extract container image name from build logs (pull)
id: extract-image
env:
# Refernce: https://docs.github.com/en/actions/learn-github-actions/expressions#example
COMMIT_STATUS_JSON: "${{ github.event_name == 'push' && steps.push-wait-build.outputs.json || steps.pull-wait-build.outputs.json }}"
run: |
./scripts/extract_image_from_build_logs.sh
# export value to job output
export IMAGE_NAME="$(cat image.name)"
echo "IMAGE_NAME: ${IMAGE_NAME}"
echo "image_name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
nats:
runs-on: ubuntu-latest
needs: wait-until-build-succeeds
env:
MANAGER_IMAGE: ${{ needs.wait-until-build-succeeds.outputs.image-name }}

steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -100,6 +142,8 @@ jobs:
PeerAuthentication:
runs-on: ubuntu-latest
needs: wait-until-build-succeeds
env:
MANAGER_IMAGE: ${{ needs.wait-until-build-succeeds.outputs.image-name }}

steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -136,3 +180,13 @@ jobs:
if: failure()
run: |
kubectl get peerauthentications.security.istio.io -A -o yaml
backend-switching:
needs: wait-until-build-succeeds
if: github.event_name == 'push'
uses: "./.github/workflows/e2e-backend-switching-reuseable.yml"
with:
eventing-manager-image: ${{ needs.wait-until-build-succeeds.outputs.image-name }}
kube-version: "1.26.9"
cluster-name-prefix: "ghem-"
secrets: inherit
53 changes: 53 additions & 0 deletions hack/ci/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,26 @@ get-nats-via-lifecycle-manager: install-lifecycle-manager install-latest-nats-mo

.PHONY: install-nats-module
install-nats-module:
$(eval LATEST_NATS_VERSION := $(shell curl -s https://api.github.com/repos/kyma-project/nats-manager/releases/latest | jq -r '.tag_name'))
@echo "Deploying NATS Manager: ${LATEST_NATS_VERSION}"
kubectl apply -f https://github.com/kyma-project/nats-manager/releases/latest/download/nats-manager.yaml
kubectl apply -f https://github.com/kyma-project/nats-manager/releases/latest/download/nats-default-cr.yaml

.PHONY: install-istio-module
install-istio-module:
$(eval LATEST_ISTIO_VERSION := $(shell curl -s https://api.github.com/repos/kyma-project/istio-manager/releases/latest | jq -r '.tag_name'))
@echo "Deploying Istio Manager: ${LATEST_ISTIO_VERSION}"
kubectl label namespace kyma-system istio-injection=enabled --overwrite
kubectl apply -f https://github.com/kyma-project/istio/releases/latest/download/istio-manager.yaml
kubectl apply -f https://github.com/kyma-project/istio/releases/latest/download/istio-default-cr.yaml

.PHONY: install-api-gateway-module
install-api-gateway-module:
$(eval LATEST_API_GATEWAY_VERSION := $(shell curl -s https://api.github.com/repos/kyma-project/api-gateway/releases/latest | jq -r '.tag_name'))
@echo "Deploying API Gateway module: ${LATEST_API_GATEWAY_VERSION}"
kubectl apply -f https://github.com/kyma-project/api-gateway/releases/latest/download/api-gateway-manager.yaml
kubectl apply -f https://github.com/kyma-project/api-gateway/releases/latest/download/apigateway-default-cr.yaml

.PHONY: verify-kyma
verify-kyma: ## Wait for Kyma CR to be in Ready state.
../verify_kyma_status.sh
Expand All @@ -88,3 +105,39 @@ install-k3d-tools: ## Create k3d with kyma CRDs.
.PHONY: apply-peerauthentication-crd
apply-peerauthentication-crd:
kubectl apply -f ../../config/crd/for-tests/security.istio.io_peerauthentication.yaml

.PHONY: wait-nats-cr-ready
wait-nats-cr-ready:
kubectl wait nats.operator.kyma-project.io -n kyma-system eventing-nats --timeout=300s --for=jsonpath='{.status.state}'=Ready

.PHONY: wait-eventing-cr-ready
wait-eventing-cr-ready:
kubectl wait eventing.operator.kyma-project.io -n kyma-system eventing --timeout=300s --for=jsonpath='{.status.state}'=Ready

.PHONY: wait-eventing-cr-ready-with-backend
wait-eventing-cr-ready-with-backend:
kubectl wait eventing.operator.kyma-project.io -n kyma-system eventing --timeout=300s --for=jsonpath='{.status.state}'=Ready --for=jsonpath='{.status.activeBackend}'=${ACTIVE_BACKEND}

.PHONY: wait-istio-cr-ready
wait-istio-cr-ready:
kubectl wait istios.operator.kyma-project.io -n kyma-system default --timeout=300s --for=jsonpath='{.status.state}'=Ready

.PHONY: wait-api-gateway-cr-ready
wait-api-gateway-cr-ready:
kubectl wait apigateways.operator.kyma-project.io -n kyma-system default --timeout=300s --for=jsonpath='{.status.state}'=Ready

.PHONY: provision-gardener-cluster
provision-gardener-cluster:
PROJECT_ROOT="${PROJECT_ROOT}" KYMA_CLI="${KYMA_CLI}" ${PROJECT_ROOT}/scripts/gardener/aws/provision.sh

.PHONY: deprovision-gardener-cluster
deprovision-gardener-cluster:
PROJECT_ROOT="${PROJECT_ROOT}" ${PROJECT_ROOT}/scripts/gardener/aws/deprovision.sh

.PHONY: create-ias-app
create-ias-app:
PROJECT_ROOT="${PROJECT_ROOT}" ${PROJECT_ROOT}/scripts/ias/create-ias-app.sh

.PHONY: delete-ias-app
delete-ias-app:
PROJECT_ROOT="${PROJECT_ROOT}" ${PROJECT_ROOT}/scripts/ias/delete-ias-app.sh
Loading

0 comments on commit 44ff053

Please sign in to comment.