Skip to content

Commit

Permalink
Polish deployment creation, add deletion validation
Browse files Browse the repository at this point in the history
Signed-off-by: Kyle Squizzato <[email protected]>
  • Loading branch information
squizzi committed Aug 23, 2024
1 parent 4496f40 commit 6ae1adc
Show file tree
Hide file tree
Showing 6 changed files with 387 additions and 140 deletions.
75 changes: 34 additions & 41 deletions test/utils/deployment.go → test/deployment/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,21 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package utils
package deployment

import (
_ "embed"
"encoding/json"
"fmt"
"os"
"os/exec"

"github.com/Mirantis/hmc/test/utils"
"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gopkg.in/yaml.v3"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

type ProviderType string
Expand All @@ -36,39 +40,32 @@ type Template string
const (
TemplateAWSStandaloneCP Template = "aws-standalone-cp"
TemplateAWSHostedCP Template = "aws-hosted-cp"

deploymentConfigFile = "./config/dev/deployment.yaml"
)

// ConfigureDeploymentConfig modifies the config/dev/deployment.yaml for
// use in test and returns the generated cluster name.
func ConfigureDeploymentConfig(provider ProviderType, templateName Template) (string, error) {
generatedName := uuid.NewString()[:8] + "-e2e-test"
//go:embed resources/deployment.yaml.tpl
var deploymentConfigBytes []byte

deploymentConfigBytes, err := os.ReadFile(deploymentConfigFile)
if err != nil {
return "", fmt.Errorf("failed to read deployment config: %w", err)
}
// GetUnstructuredDeployment returns an unstructured deployment object based on
// the provider and template.
func GetUnstructuredDeployment(provider ProviderType, templateName Template) *unstructured.Unstructured {
GinkgoHelper()

generatedName := uuid.New().String()[:8] + "-e2e-test"
_, _ = fmt.Fprintf(GinkgoWriter, "Generated AWS cluster name: %q\n", generatedName)

var deploymentConfig map[string]interface{}

err = yaml.Unmarshal(deploymentConfigBytes, &deploymentConfig)
if err != nil {
return "", fmt.Errorf("failed to unmarshal deployment config: %w", err)
}
err := yaml.Unmarshal(deploymentConfigBytes, &deploymentConfig)
Expect(err).NotTo(HaveOccurred(), "failed to unmarshal deployment config")

switch provider {
case ProviderAWS:
// XXX: Maybe we should just use automatic AMI selection here.
amiID, err := getAWSAMI()
if err != nil {
return "", fmt.Errorf("failed to get AWS AMI: %w", err)
}

amiID := getAWSAMI()
awsRegion := os.Getenv("AWS_REGION")

// Modify the existing ./config/dev/deployment.yaml file to use the
// AMI we just found and our AWS_REGION.
// Modify the deployment config to use the generated name and the AMI.
// TODO: This should be modified to use go templating.
if metadata, ok := deploymentConfig["metadata"].(map[string]interface{}); ok {
metadata["name"] = generatedName
} else {
Expand All @@ -92,34 +89,28 @@ func ConfigureDeploymentConfig(provider ProviderType, templateName Template) (st
}
}

deploymentConfigBytes, err = yaml.Marshal(deploymentConfig)
if err != nil {
return "", fmt.Errorf("failed to marshal deployment config: %w", err)
}

_, _ = fmt.Fprintf(GinkgoWriter, "Generated AWS cluster name: %q\n", generatedName)

return generatedName, os.WriteFile(deploymentConfigFile, deploymentConfigBytes, 0644)
return &unstructured.Unstructured{Object: deploymentConfig}
default:
return "", fmt.Errorf("unsupported provider: %s", provider)
Fail(fmt.Sprintf("unsupported provider: %s", provider))
}

return nil
}

// getAWSAMI returns an AWS AMI ID to use for test.
func getAWSAMI() (string, error) {
func getAWSAMI() string {
GinkgoHelper()

// For now we'll just use the latest Kubernetes version for ubuntu 20.04,
// but we could potentially pin the Kube version and specify that here.
cmd := exec.Command("./bin/clusterawsadm", "ami", "list", "--os=ubuntu-20.04", "-o", "json")
output, err := Run(cmd)
if err != nil {
return "", fmt.Errorf("failed to list AMIs: %w", err)
}
output, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "failed to list AMIs")

var amiList map[string]interface{}

if err := json.Unmarshal(output, &amiList); err != nil {
return "", fmt.Errorf("failed to unmarshal AMI list: %w", err)
}
err = json.Unmarshal(output, &amiList)
Expect(err).NotTo(HaveOccurred(), "failed to unmarshal AMI list")

// ami list returns a sorted list of AMIs by kube version, just get the
// first one.
Expand All @@ -131,9 +122,11 @@ func getAWSAMI() (string, error) {
continue
}

return ami, nil
return ami
}
}

return "", fmt.Errorf("no AMIs found")
Fail("no AMIs found")

return ""
}
17 changes: 17 additions & 0 deletions test/deployment/resources/deployment.yaml.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
apiVersion: hmc.mirantis.com/v1alpha1
kind: Deployment
metadata:
name: ${DEPLOYMENT_NAME}
spec:
config:
controlPlane:
amiID: ${AMI_ID}
instanceType: ${INSTANCE_TYPE}
controlPlaneNumber: ${CONTROL_PLANE_NUMBER}
publicIP: ${PUBLIC_IP}
region: ${AWS_REGION}
worker:
amiID: ${AMI_ID}
instanceType: ${INSTANCE_TYPE}
workersNumber: ${WORKERS_NUMBER}
template: ${TEMPLATE_NAME}
116 changes: 116 additions & 0 deletions test/deployment/validate_deleted.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
// Copyright 2024
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package deployment

import (
"context"
"fmt"

"github.com/Mirantis/hmc/test/kubeclient"
. "github.com/onsi/ginkgo/v2"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

var deletionValidators = map[string]resourceValidationFunc{
"clusters": validateClusterDeleted,
"machinedeployments": validateMachineDeploymentsDeleted,
"control-planes": validateK0sControlPlanesDeleted,
}

// VerifyProviderDeleted is a provider-agnostic verification that checks
// to ensure generic resources managed by the provider have been deleted.
// It is intended to be used in conjunction with an Eventually block.
func VerifyProviderDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
// Sequentially validate each resource type, only returning the first error
// as to not move on to the next resource type until the first is resolved.
// We use []string here since order is important.
for _, name := range []string{"control-planes", "machinedeployments", "clusters"} {
validator, ok := resourceValidators[name]
if !ok {
continue
}

if err := validator(ctx, kc, clusterName); err != nil {
_, _ = fmt.Fprintf(GinkgoWriter, "[%s] validation error: %v\n", name, err)
return err
}

_, _ = fmt.Fprintf(GinkgoWriter, "[%s] validation succeeded\n", name)
delete(resourceValidators, name)
}

return nil
}

// validateClusterDeleted validates that the Cluster resource has been deleted.
func validateClusterDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
// Validate that the Cluster resource has been deleted
cluster, err := kc.GetCluster(ctx, clusterName)
if err != nil {
return err
}

var inPhase string

if cluster != nil {
phase, _, _ := unstructured.NestedString(cluster.Object, "status", "phase")
if phase != "" {
inPhase = ", in phase: " + phase
}

return fmt.Errorf("cluster %q still exists%s", clusterName, inPhase)
}

return nil
}

// validateMachineDeploymentsDeleted validates that all MachineDeployments have
// been deleted.
func validateMachineDeploymentsDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
machineDeployments, err := kc.ListMachineDeployments(ctx, clusterName)
if err != nil {
return err
}

var mdNames []string
if len(machineDeployments) > 0 {
for _, md := range machineDeployments {
mdNames = append(mdNames, md.GetName())

return fmt.Errorf("machine deployments still exist: %s", mdNames)
}
}

return nil
}

// validateK0sControlPlanesDeleted validates that all k0scontrolplanes have
// been deleted.
func validateK0sControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
controlPlanes, err := kc.ListK0sControlPlanes(ctx, clusterName)
if err != nil {
return err
}

var cpNames []string
if len(controlPlanes) > 0 {
for _, cp := range controlPlanes {
cpNames = append(cpNames, cp.GetName())

return fmt.Errorf("k0s control planes still exist: %s", cpNames)
}
}

return nil
}
Loading

0 comments on commit 6ae1adc

Please sign in to comment.