Skip to content

Commit

Permalink
[e2e] Support to provide e2e testing configuration
Browse files Browse the repository at this point in the history
Closes #641
  • Loading branch information
eromanova committed Dec 12, 2024
1 parent 7ecf348 commit 252ee03
Show file tree
Hide file tree
Showing 20 changed files with 757 additions and 427 deletions.
20 changes: 10 additions & 10 deletions .github/workflows/build_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.vars.outputs.version }}
clustername: ${{ steps.vars.outputs.clustername }}
clusterprefix: ${{ steps.vars.outputs.clusterprefix }}
pr: ${{ steps.pr.outputs.result }}
steps:
- name: Get PR ref
Expand Down Expand Up @@ -72,7 +72,7 @@ jobs:
run: |
GIT_VERSION=$(git describe --tags --always)
echo "version=${GIT_VERSION:1}" >> $GITHUB_OUTPUT
echo "clustername=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT
echo "clusterprefix=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT
- name: Build and push HMC controller image
uses: docker/build-push-action@v6
with:
Expand All @@ -98,7 +98,7 @@ jobs:
group: controller-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
outputs:
clustername: ${{ needs.build.outputs.clustername }}
clusterprefix: ${{ needs.build.outputs.clusterprefix }}
version: ${{ needs.build.outputs.version }}
pr: ${{ needs.build.outputs.pr }}
steps:
Expand All @@ -112,7 +112,7 @@ jobs:
- name: Run E2E tests
env:
GINKGO_LABEL_FILTER: 'controller'
MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
MANAGED_CLUSTER_PREFIX: ${{ needs.build.outputs.clusterprefix }}
IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
VERSION: ${{ needs.build.outputs.version }}
run: |
Expand All @@ -134,7 +134,7 @@ jobs:
group: cloud-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
outputs:
clustername: ${{ needs.build.outputs.clustername }}
clusterprefix: ${{ needs.build.outputs.clusterprefix }}
version: ${{ needs.build.outputs.version }}
pr: ${{ needs.build.outputs.pr }}
env:
Expand Down Expand Up @@ -162,7 +162,7 @@ jobs:
- name: Run E2E tests
env:
GINKGO_LABEL_FILTER: 'provider:cloud'
MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
MANAGED_CLUSTER_PREFIX: ${{ needs.build.outputs.clusterprefix }}
IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
VERSION: ${{ needs.build.outputs.version }}
run: |
Expand All @@ -184,7 +184,7 @@ jobs:
group: onprem-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
outputs:
clustername: ${{ needs.build.outputs.clustername }}
clusterprefix: ${{ needs.build.outputs.clusterprefix }}
version: ${{ needs.build.outputs.version }}
pr: ${{ needs.build.outputs.pr }}
env:
Expand Down Expand Up @@ -215,7 +215,7 @@ jobs:
- name: Run E2E tests
env:
GINKGO_LABEL_FILTER: 'provider:onprem'
MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
MANAGED_CLUSTER_PREFIX: ${{ needs.build.outputs.clusterprefix }}
IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
VERSION: ${{ needs.build.outputs.version }}
run: |
Expand All @@ -237,7 +237,7 @@ jobs:
if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.build.result, 'success') }}
timeout-minutes: 15
outputs:
clustername: ${{ needs.build.outputs.clustername }}
clusterprefix: ${{ needs.build.outputs.clusterprefix }}
version: ${{ needs.build.outputs.version }}
pr: ${{ needs.build.outputs.pr }}
steps:
Expand All @@ -260,7 +260,7 @@ jobs:
AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }}
CLUSTER_NAME: '${{ needs.build.outputs.clustername }}'
CLUSTER_NAME: '${{ needs.build.outputs.clusterprefix }}'
run: |
make dev-aws-nuke
make dev-azure-nuke
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,18 @@ tidy:
test: generate-all fmt vet envtest tidy external-crd ## Run tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out

# E2E_CONFIG_B64 contains the configuration for e2e testing.
E2E_CONFIG_B64 ?= ""

# Utilize Kind or modify the e2e tests to load the image locally, enabling
# compatibility with other vendors.
.PHONY: test-e2e # Run the e2e tests using a Kind k8s instance as the management cluster.
test-e2e: cli-install
@if [ "$$GINKGO_LABEL_FILTER" ]; then \
ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \
fi; \
KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag
KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) E2E_CONFIG_B64=$(E2E_CONFIG_B64) \
go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag

.PHONY: lint
lint: golangci-lint ## Run golangci-lint linter & yamllint
Expand Down
10 changes: 5 additions & 5 deletions docs/dev.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,13 +140,13 @@ IMG="ghcr.io/mirantis/hmc/controller-ci:v0.0.1-179-ga5bdf29" \
Optionally, the `NO_CLEANUP=1` env var can be used to disable `After` nodes from
running within some specs, this will allow users to debug tests by re-running
them without the need to wait a while for an infrastructure deployment to occur.
For subsequent runs the `MANAGED_CLUSTER_NAME=<cluster name>` env var should be
passed to tell the test what cluster name to use so that it does not try to
generate a new name and deploy a new cluster.
For subsequent runs the `MANAGED_CLUSTER_PREFIX=<cluster prefix>` env var should be
passed to tell the test what cluster prefix to use so that it does not try to
generate a new cluster name and deploy a new cluster.

Tests that run locally use autogenerated names like `12345678-e2e-test` while
Tests that run locally use autogenerated names prefixes like `e2e-test-12345678` while
tests that run in CI use names such as `ci-1234567890-e2e-test`. You can always
pass `MANAGED_CLUSTER_NAME=` from the get-go to customize the name used by the
pass `MANAGED_CLUSTER_PREFIX=` from the get-go to customize the prefix used by the
test.

### Filtering test runs
Expand Down
135 changes: 135 additions & 0 deletions test/e2e/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// Copyright 2024
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
"encoding/base64"
"fmt"
"os"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gopkg.in/yaml.v3"
)

type TestingProvider string

const (
envVarE2EConfig = "E2E_CONFIG_B64"

TestingProviderAWS TestingProvider = "aws"
TestingProviderAzure TestingProvider = "azure"
TestingProviderVsphere TestingProvider = "vsphere"
)

var (
Config TestingConfig

defaultConfig = map[TestingProvider][]ProviderTestingConfig{
TestingProviderAWS: {},
TestingProviderAzure: {},
TestingProviderVsphere: {},
}

defaultStandaloneTemplates = map[TestingProvider]string{
TestingProviderAWS: "aws-standalone-cp-0-0-3",
TestingProviderAzure: "azure-standalone-cp-0-0-3",
TestingProviderVsphere: "vsphere-standalone-cp-0-0-3",
}

defaultHostedTemplates = map[TestingProvider]string{
TestingProviderAWS: "aws-hosted-cp-0-0-3",
TestingProviderAzure: "azure-hosted-cp-0-0-3",
TestingProviderVsphere: "vsphere-hosted-cp-0-0-3",
}
)

type TestingConfig = map[TestingProvider][]ProviderTestingConfig

type ProviderTestingConfig struct {
// Standalone contains the testing configuration for the standalone cluster deployment.
Standalone *ClusterTestingConfig `yaml:"standalone,omitempty"`
// Standalone contains the testing configuration for the hosted cluster deployment.
Hosted *ClusterTestingConfig `yaml:"hosted,omitempty"`
}

type ClusterTestingConfig struct {
// Upgrade is a boolean parameter that specifies whether the managed cluster upgrade should be tested.
Upgrade bool `yaml:"upgrade,omitempty"`
// Template is the name of the template to use when deploying a managed cluster.
// If unset:
// * The latest available template will be chosen
// * If upgrade is triggered, the latest available template with available upgrades will be chosen.
Template string `yaml:"template,omitempty"`
// UpgradeTemplate specifies the name of the template to upgrade to. Ignored if upgrade is set to false.
// If unset, the latest template available for the upgrade will be chosen.
UpgradeTemplate string `yaml:"upgradeTemplate,omitempty"`
}

func Parse() error {
decodedConfig, err := base64.StdEncoding.DecodeString(os.Getenv(envVarE2EConfig))
if err != nil {
return err
}

err = yaml.Unmarshal(decodedConfig, &Config)
if err != nil {
return err
}

setDefaults()
_, _ = fmt.Fprintf(GinkgoWriter, "E2e testing configuration:\n%s\n", Show())
return nil
}

func setDefaults() {
if len(Config) == 0 {
Config = defaultConfig
}
for provider, configs := range Config {
if len(configs) == 0 {
Config[provider] = []ProviderTestingConfig{
{
Standalone: &ClusterTestingConfig{},
Hosted: &ClusterTestingConfig{},
},
}
}
for i := range Config[provider] {
config := Config[provider][i]
if config.Standalone != nil && config.Standalone.Template == "" {
config.Standalone.Template = defaultStandaloneTemplates[provider]
}
if config.Hosted != nil && config.Hosted.Template == "" {
config.Hosted.Template = defaultHostedTemplates[provider]
}
Config[provider][i] = config
}
}
}

func Show() string {
prettyConfig, err := yaml.Marshal(Config)
Expect(err).NotTo(HaveOccurred())

return string(prettyConfig)
}

func (c *ProviderTestingConfig) String() string {
prettyConfig, err := yaml.Marshal(c)
Expect(err).NotTo(HaveOccurred())

return string(prettyConfig)
}
Loading

0 comments on commit 252ee03

Please sign in to comment.