Skip to content

Commit

Permalink
Adding more integration tests
Browse files Browse the repository at this point in the history
Signed-off-by: Erica Jin <[email protected]>
  • Loading branch information
jaswalkiranavtar authored and EricaJ6 committed Nov 22, 2024
1 parent efb1c79 commit 2ab4069
Show file tree
Hide file tree
Showing 8 changed files with 245 additions and 72 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ metadata:
"{{ $key }}": "{{ $value }}"
{{ end }}
{{ end }}
{{ if and .ManagedClusterRoleArn (eq .RegistrationDriver.AuthType "awsirsa") }}
{{ if and .ManagedClusterRoleArn (eq .RegistrationDriver.AuthType "awsirsa") }}
annotations:
eks.amazonaws.com/role-arn: {{ .ManagedClusterRoleArn }}
{{ end }}
{{ end }}
imagePullSecrets:
- name: open-cluster-management-image-pull-credentials
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ metadata:
"{{ $key }}": "{{ $value }}"
{{ end }}
{{ end }}
{{ if and .ManagedClusterRoleArn (eq .RegistrationDriver.AuthType "awsirsa") }}
{{ if and .ManagedClusterRoleArn (eq .RegistrationDriver.AuthType "awsirsa") }}
annotations:
eks.amazonaws.com/role-arn: {{ .ManagedClusterRoleArn }}
{{ end }}
{{ end }}
imagePullSecrets:
- name: open-cluster-management-image-pull-credentials
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"crypto/md5" // #nosec G501
"encoding/hex"
er "errors"
"fmt"
"strings"
"time"
Expand Down Expand Up @@ -344,13 +343,13 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
if !commonhelpers.IsEksArnWellFormed(hubClusterArn) {
errorMsg := fmt.Sprintf("HubClusterArn %s is not well formed", hubClusterArn)
klog.Errorf(errorMsg)
return er.New(errorMsg)
return fmt.Errorf(errorMsg)
}

if !commonhelpers.IsEksArnWellFormed(managedClusterArn) {
errorMsg := fmt.Sprintf("ManagedClusterArn %s is not well formed", managedClusterArn)
klog.Errorf(errorMsg)
return er.New(errorMsg)
return fmt.Errorf(errorMsg)
}

config.RegistrationDriver = RegistrationDriver{
Expand Down
174 changes: 174 additions & 0 deletions test/integration/operator/klusterlet_aws_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
package operator

import (
"context"
"fmt"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"

operatorapiv1 "open-cluster-management.io/api/operator/v1"

"open-cluster-management.io/ocm/pkg/operator/helpers"
"open-cluster-management.io/ocm/pkg/registration/spoke"
"open-cluster-management.io/ocm/test/integration/util"
)

var _ = ginkgo.Describe("Klusterlet using aws auth", func() {
var cancel context.CancelFunc
var klusterlet *operatorapiv1.Klusterlet
var hubKubeConfigSecret *corev1.Secret
var klusterletNamespace string
var registrationDeploymentName string
var registrationSAName string
var workDeploymentName string
var workSAName string
var agentLabelSelector string

ginkgo.BeforeEach(func() {
var ctx context.Context

klusterletNamespace = fmt.Sprintf("open-cluster-management-aws-%s", rand.String(6))
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: klusterletNamespace,
},
}
_, err := kubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

klusterlet = &operatorapiv1.Klusterlet{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("klusterlet-%s", rand.String(6)),
Labels: map[string]string{"test": "123", "component": "klusterlet", "123": "312"},
},
Spec: operatorapiv1.KlusterletSpec{
RegistrationImagePullSpec: "quay.io/open-cluster-management/registration",
WorkImagePullSpec: "quay.io/open-cluster-management/work",
ExternalServerURLs: []operatorapiv1.ServerURL{
{
URL: "https://localhost",
},
},
ClusterName: "testcluster",
Namespace: klusterletNamespace,
RegistrationConfiguration: &operatorapiv1.RegistrationConfiguration{
RegistrationDriver: operatorapiv1.RegistrationDriver{
AuthType: spoke.AwsIrsaAuthType,
AwsIrsa: &operatorapiv1.AwsIrsa{
HubClusterArn: util.HubClusterArn,
ManagedClusterArn: util.ManagedClusterArn,
},
},
},
},
}

agentLabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{
MatchLabels: helpers.GetKlusterletAgentLabels(klusterlet),
})

hubKubeConfigSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: helpers.HubKubeConfig,
Namespace: klusterletNamespace,
},
Data: map[string][]byte{
"placeholder": []byte("placeholder"),
},
}
_, err = kubeClient.CoreV1().Secrets(klusterletNamespace).Create(context.Background(), hubKubeConfigSecret, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

ctx, cancel = context.WithCancel(context.Background())
go startKlusterletOperator(ctx)
})

ginkgo.AfterEach(func() {
err := kubeClient.CoreV1().Namespaces().Delete(context.Background(), klusterletNamespace, metav1.DeleteOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())

if cancel != nil {
cancel()
}
})

ginkgo.Context("Deploy and clean klusterlet component using aws auth", func() {
ginkgo.BeforeEach(func() {
registrationDeploymentName = fmt.Sprintf("%s-registration-agent", klusterlet.Name)
workDeploymentName = fmt.Sprintf("%s-work-agent", klusterlet.Name)

registrationSAName = fmt.Sprintf("%s-registration-sa", klusterlet.Name)
workSAName = fmt.Sprintf("%s-work-sa", klusterlet.Name)
})

ginkgo.AfterEach(func() {
gomega.Expect(operatorClient.OperatorV1().Klusterlets().Delete(context.Background(), klusterlet.Name, metav1.DeleteOptions{})).To(gomega.BeNil())
})

ginkgo.It("should have expected resource created successfully using aws auth", func() {
_, err := operatorClient.OperatorV1().Klusterlets().Create(context.Background(), klusterlet, metav1.CreateOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())

// Check service account
gomega.Eventually(func() bool {
serviceaccouts, err := kubeClient.CoreV1().ServiceAccounts(klusterletNamespace).List(context.Background(),
metav1.ListOptions{LabelSelector: agentLabelSelector})
if err != nil {
return false
}
if len(serviceaccouts.Items) != 2 {
return false
}
for _, serviceAccount := range serviceaccouts.Items {
if serviceAccount.GetName() != registrationSAName &&
serviceAccount.GetName() != workSAName {
return false
}
if serviceAccount.ObjectMeta.Annotations[util.IrsaAnnotationKey] != util.PrerequisiteSpokeRoleArn {
return false
}
}
return true
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

// Check deployment
gomega.Eventually(func() bool {
deployments, err := kubeClient.AppsV1().Deployments(klusterletNamespace).List(context.Background(),
metav1.ListOptions{LabelSelector: agentLabelSelector})
if err != nil {
return false
}
if len(deployments.Items) != 2 {
return false
}

for _, deployment := range deployments.Items {
if deployment.GetName() != registrationDeploymentName &&
deployment.GetName() != workDeploymentName {
return false
}
if deployment.GetName() == registrationDeploymentName {
if !util.AllCommandLineOptionsPresent(deployment) || !util.AwsCliSpecificVolumesMounted(deployment) {
return false
}
}
if deployment.GetName() == workDeploymentName {
if !util.AwsCliSpecificVolumesMounted(deployment) {
return false
}
}
}

return true
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

util.AssertKlusterletCondition(klusterlet.Name, operatorClient, "Applied", "KlusterletApplied", metav1.ConditionTrue)
})

})

})
45 changes: 7 additions & 38 deletions test/integration/operator/klusterlet_singleton_aws_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package operator
import (
"context"
"fmt"
"strings"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
Expand All @@ -14,15 +13,10 @@ import (
operatorapiv1 "open-cluster-management.io/api/operator/v1"

"open-cluster-management.io/ocm/pkg/operator/helpers"
"open-cluster-management.io/ocm/pkg/registration/spoke"
"open-cluster-management.io/ocm/test/integration/util"
)

var hubClusterArn string = "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1"
var managedClusterArn string = "arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1"
var managedClusterRoleSuffix string = "7f8141296c75f2871e3d030f85c35692"
var prerequisiteSpokeRoleArn string = "arn:aws:iam::123456789012:role/ocm-managed-cluster-" + managedClusterRoleSuffix
var irsaAnnotationKey string = "eks.amazonaws.com/role-arn"

var _ = ginkgo.Describe("Klusterlet Singleton mode with aws auth", func() {
var cancel context.CancelFunc
var klusterlet *operatorapiv1.Klusterlet
Expand All @@ -37,7 +31,7 @@ var _ = ginkgo.Describe("Klusterlet Singleton mode with aws auth", func() {
Name: fmt.Sprintf("klusterlet-%s", rand.String(6)),
},
Spec: operatorapiv1.KlusterletSpec{
Namespace: fmt.Sprintf("%s-aws", helpers.KlusterletDefaultNamespace),
Namespace: fmt.Sprintf("%s-singleton-aws", helpers.KlusterletDefaultNamespace),
ImagePullSpec: "quay.io/open-cluster-management/registration-operator",
ExternalServerURLs: []operatorapiv1.ServerURL{
{
Expand All @@ -50,10 +44,10 @@ var _ = ginkgo.Describe("Klusterlet Singleton mode with aws auth", func() {
},
RegistrationConfiguration: &operatorapiv1.RegistrationConfiguration{
RegistrationDriver: operatorapiv1.RegistrationDriver{
AuthType: "awsirsa",
AuthType: spoke.AwsIrsaAuthType,
AwsIrsa: &operatorapiv1.AwsIrsa{
HubClusterArn: hubClusterArn,
ManagedClusterArn: managedClusterArn,
HubClusterArn: util.HubClusterArn,
ManagedClusterArn: util.ManagedClusterArn,
},
},
},
Expand Down Expand Up @@ -101,7 +95,7 @@ var _ = ginkgo.Describe("Klusterlet Singleton mode with aws auth", func() {
if err != nil {
return false
}
return sa.ObjectMeta.Annotations[irsaAnnotationKey] == prerequisiteSpokeRoleArn
return sa.ObjectMeta.Annotations[util.IrsaAnnotationKey] == util.PrerequisiteSpokeRoleArn
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

// Check deployment
Expand All @@ -110,32 +104,7 @@ var _ = ginkgo.Describe("Klusterlet Singleton mode with aws auth", func() {
if err != nil {
return false
}

isRegistrationAuthPresent := false
isManagedClusterArnPresent := false
isManagedClusterRoleSuffixPresent := false
for _, arg := range deployment.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(arg, "--registration-auth=awsirsa") {
isRegistrationAuthPresent = true
}
if strings.Contains(arg, "--managed-cluster-arn=arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1") {
isManagedClusterArnPresent = true
}
if strings.Contains(arg, "--managed-cluster-role-suffix="+managedClusterRoleSuffix) {
isManagedClusterRoleSuffixPresent = true
}
}
allCommandLineOptionsPresent := isRegistrationAuthPresent && isManagedClusterArnPresent && isManagedClusterRoleSuffixPresent

isDotAwsMounted := false
for _, volumeMount := range deployment.Spec.Template.Spec.Containers[0].VolumeMounts {
if volumeMount.Name == "dot-aws" && volumeMount.MountPath == "/.aws" {
isDotAwsMounted = true
}
}

awsCliSpecificVolumesMounted := isDotAwsMounted
return allCommandLineOptionsPresent && awsCliSpecificVolumesMounted
return util.AllCommandLineOptionsPresent(*deployment) && util.AwsCliSpecificVolumesMounted(*deployment)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

util.AssertKlusterletCondition(klusterlet.Name, operatorClient, "Applied", "KlusterletApplied", metav1.ConditionTrue)
Expand Down
29 changes: 2 additions & 27 deletions test/integration/operator/klusterlet_singleton_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package operator
import (
"context"
"fmt"
"strings"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
Expand Down Expand Up @@ -190,7 +189,7 @@ var _ = ginkgo.Describe("Klusterlet Singleton mode", func() {
if err != nil {
return false
}
return sa.ObjectMeta.Annotations[irsaAnnotationKey] != prerequisiteSpokeRoleArn
return sa.ObjectMeta.Annotations[util.IrsaAnnotationKey] != util.PrerequisiteSpokeRoleArn
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

// Check deployment
Expand All @@ -200,31 +199,7 @@ var _ = ginkgo.Describe("Klusterlet Singleton mode", func() {
return false
}

isRegistrationAuthPresent := false
isManagedClusterArnPresent := false
isManagedClusterRoleSuffixPresent := false
for _, arg := range deployment.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(arg, "--registration-auth=awsirsa") {
isRegistrationAuthPresent = true
}
if strings.Contains(arg, "--managed-cluster-arn=arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1") {
isManagedClusterArnPresent = true
}
if strings.Contains(arg, "--managed-cluster-role-suffix="+managedClusterRoleSuffix) {
isManagedClusterRoleSuffixPresent = true
}
}
anyCommandLineOptionsPresent := isRegistrationAuthPresent || isManagedClusterArnPresent || isManagedClusterRoleSuffixPresent

isDotAwsMounted := false
for _, volumeMount := range deployment.Spec.Template.Spec.Containers[0].VolumeMounts {
if volumeMount.Name == "dot-aws" && volumeMount.MountPath == "/.aws" {
isDotAwsMounted = true
}
}

awsCliSpecificVolumesMounted := isDotAwsMounted
return !(anyCommandLineOptionsPresent || awsCliSpecificVolumesMounted)
return !util.AllCommandLineOptionsPresent(*deployment) && !util.AwsCliSpecificVolumesMounted(*deployment)
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

// Check addon namespace
Expand Down
13 changes: 13 additions & 0 deletions test/integration/operator/klusterlet_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,9 @@ var _ = ginkgo.Describe("Klusterlet", func() {
serviceAccount.GetName() != workSAName {
return false
}
if serviceAccount.ObjectMeta.Annotations[util.IrsaAnnotationKey] == util.PrerequisiteSpokeRoleArn {
return false
}
}
return true
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
Expand All @@ -289,6 +292,16 @@ var _ = ginkgo.Describe("Klusterlet", func() {
deployment.GetName() != workDeploymentName {
return false
}
if deployment.GetName() == registrationDeploymentName {
if util.AllCommandLineOptionsPresent(deployment) || util.AwsCliSpecificVolumesMounted(deployment) {
return false
}
}
if deployment.GetName() == workDeploymentName {
if util.AwsCliSpecificVolumesMounted(deployment) {
return false
}
}
}
return true
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
Expand Down
Loading

0 comments on commit 2ab4069

Please sign in to comment.