From c5f45661d83fbbe66c9e645e7c2b73c7d8add591 Mon Sep 17 00:00:00 2001 From: Zijun Wang Date: Fri, 29 Sep 2023 12:23:22 -0700 Subject: [PATCH] Create VpcAssociationPolicy e2e test --- test/pkg/test/framework.go | 24 +-- test/suites/integration/suite_test.go | 13 +- .../vpc_association_policy_test.go | 145 ++++++++++++++++++ 3 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 test/suites/integration/vpc_association_policy_test.go diff --git a/test/pkg/test/framework.go b/test/pkg/test/framework.go index acc524e4..fbed1778 100644 --- a/test/pkg/test/framework.go +++ b/test/pkg/test/framework.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/onsi/gomega/format" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -84,12 +85,15 @@ func addOptionalCRDs(scheme *runtime.Scheme) { scheme.AddKnownTypes(dnsEndpoint, &endpoint.DNSEndpoint{}, &endpoint.DNSEndpointList{}) metav1.AddToGroupVersion(scheme, dnsEndpoint) - targetGroupPolicy := schema.GroupVersion{ - Group: "application-networking.k8s.aws", + awsGatewayControllerCRDGroupVersion := schema.GroupVersion{ + Group: v1alpha1.GroupName, Version: "v1alpha1", } - scheme.AddKnownTypes(targetGroupPolicy, &v1alpha1.TargetGroupPolicy{}, &v1alpha1.TargetGroupPolicyList{}) - metav1.AddToGroupVersion(scheme, targetGroupPolicy) + scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.TargetGroupPolicy{}, &v1alpha1.TargetGroupPolicyList{}) + metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion) + + scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.VpcAssociationPolicy{}, &v1alpha1.VpcAssociationPolicyList{}) + metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion) } type Framework struct { @@ -100,6 +104,7 @@ type Framework struct { namespace string controllerRuntimeConfig *rest.Config LatticeClient services.Lattice + Ec2Client *ec2.EC2 GrpcurlRunner *v1.Pod } @@ -111,6 +116,7 @@ func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) * framework := &Framework{ Client: lo.Must(client.New(controllerRuntimeConfig, client.Options{Scheme: testScheme})), LatticeClient: services.NewDefaultLattice(session.Must(session.NewSession()), config.Region), // region is currently hardcoded + Ec2Client: ec2.New(session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)}))), GrpcurlRunner: &v1.Pod{}, ctx: ctx, log: log, @@ -362,23 +368,23 @@ func (env *Framework) VerifyTargetGroupNotFound(tg *vpclattice.TargetGroupSummar }).Should(Succeed()) } -func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpcId string, serviceNetwork *vpclattice.ServiceNetworkSummary) (bool, error) { +func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpcId string, serviceNetwork *vpclattice.ServiceNetworkSummary) (bool, string, error) { env.log.Infof("IsVpcAssociatedWithServiceNetwork vpcId:%v serviceNetwork: %v \n", vpcId, serviceNetwork) vpcAssociations, err := env.LatticeClient.ListServiceNetworkVpcAssociationsAsList(ctx, &vpclattice.ListServiceNetworkVpcAssociationsInput{ ServiceNetworkIdentifier: serviceNetwork.Id, VpcIdentifier: &vpcId, }) if err != nil { - return false, err + return false, "", err } if len(vpcAssociations) != 1 { - return false, fmt.Errorf("Expect to have one VpcServiceNetworkAssociation len(vpcAssociations): %d", len(vpcAssociations)) + return false, "", fmt.Errorf("Expect to have one VpcServiceNetworkAssociation len(vpcAssociations): %d", len(vpcAssociations)) } association := vpcAssociations[0] if *association.Status != vpclattice.ServiceNetworkVpcAssociationStatusActive { - return false, fmt.Errorf("Current cluster should have one Active status association *association.Status: %s, err: %w", *association.Status, err) + return false, "", fmt.Errorf("Current cluster should have one Active status association *association.Status: %s, err: %w", *association.Status, err) } - return true, nil + return true, *association.Id, nil } func (env *Framework) AreAllLatticeTargetsHealthy(ctx context.Context, tg *vpclattice.TargetGroupSummary) (bool, error) { diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index bd17d592..6619f14a 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -5,6 +5,7 @@ import ( "flag" "os" + "github.com/aws/aws-sdk-go/service/vpclattice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -25,9 +26,10 @@ const ( var testFramework *test.Framework var ctx context.Context var testGateway *v1beta1.Gateway +var testServiceNetwork *vpclattice.ServiceNetworkSummary var _ = BeforeSuite(func() { - vpcid := os.Getenv("CLUSTER_VPC_ID") - if vpcid == "" { + vpcId := os.Getenv("CLUSTER_VPC_ID") + if vpcId == "" { Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests") } @@ -44,11 +46,12 @@ var _ = BeforeSuite(func() { testGateway = testFramework.NewGateway("test-gateway", k8snamespace) testFramework.ExpectCreated(ctx, testGateway) - sn := testFramework.GetServiceNetwork(ctx, testGateway) + testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway) - test.Logger(ctx).Infof("Expecting VPC %s and service network %s association", vpcid, *sn.Id) + test.Logger(ctx).Infof("Expecting VPC %s and service network %s association", vpcId, *testServiceNetwork.Id) Eventually(func(g Gomega) { - g.Expect(testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcid, sn)).To(BeTrue()) + associated, _, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcId, testServiceNetwork) + g.Expect(associated).To(BeTrue()) }).Should(Succeed()) }) diff --git a/test/suites/integration/vpc_association_policy_test.go b/test/suites/integration/vpc_association_policy_test.go new file mode 100644 index 00000000..c4025b7b --- /dev/null +++ b/test/suites/integration/vpc_association_policy_test.go @@ -0,0 +1,145 @@ +package integration + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/vpclattice" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gateway_api_v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gateway_api "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" +) + +var _ = Describe("Test vpc association policy", Ordered, Focus, func() { + var ( + vpcAssociationPolicy *v1alpha1.VpcAssociationPolicy + sgId v1alpha1.SecurityGroupId + ) + + BeforeAll(func() { + // Create security group + describeVpcOutput, err := testFramework.Ec2Client.DescribeVpcs(&ec2.DescribeVpcsInput{ + VpcIds: []*string{aws.String(test.CurrentClusterVpcId)}, + }) + Expect(err).To(BeNil()) + sourceVPC := describeVpcOutput.Vpcs[0] + createSgOutput, err := testFramework.Ec2Client.CreateSecurityGroupWithContext(ctx, &ec2.CreateSecurityGroupInput{ + Description: aws.String("k8s-test-lattice-snva-sg"), + GroupName: aws.String("k8s-test-lattice-snva-sg"), + VpcId: aws.String(test.CurrentClusterVpcId), + }) + Expect(err).To(BeNil()) + sgId = v1alpha1.SecurityGroupId(*createSgOutput.GroupId) + + // Create security group inbound rules + _, err = testFramework.Ec2Client.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: createSgOutput.GroupId, + IpPermissions: []*ec2.IpPermission{ + { // SG Rule to allow HTTP + IpProtocol: aws.String("tcp"), + IpRanges: []*ec2.IpRange{ + { + CidrIp: sourceVPC.CidrBlock, + }, + }, + FromPort: aws.Int64(80), + ToPort: aws.Int64(80), + }, + { // SG Rule to allow HTTPS + IpProtocol: aws.String("tcp"), + IpRanges: []*ec2.IpRange{ + { + CidrIp: sourceVPC.CidrBlock, + }, + }, + FromPort: aws.Int64(443), + ToPort: aws.Int64(443), + }, + }, + }) + Expect(err).To(BeNil()) + }) + + It("Create a VpcAssociationPolicy that set associateWithVpc to false, expecting do not create ServiceNetworkVpcAssociation", func() { + vpcAssociationPolicy = &v1alpha1.VpcAssociationPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vpc-association-policy", + Namespace: k8snamespace, + }, + Spec: v1alpha1.VpcAssociationPolicySpec{ + TargetRef: &gateway_api_v1alpha2.PolicyTargetReference{ + Group: gateway_api.GroupName, + Kind: "Gateway", + Name: gateway_api_v1alpha2.ObjectName(testGateway.Name), + Namespace: lo.ToPtr(gateway_api_v1alpha2.Namespace(k8snamespace)), + }, + AssociateWithVpc: lo.ToPtr(false), + }, + } + testFramework.ExpectCreated(ctx, vpcAssociationPolicy) + Eventually(func(g Gomega) { + //Expect no SNVA for testGateway + associated, _, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork) + g.Expect(err).To(Not(BeNil())) + g.Expect(associated).To(BeFalse()) + }).Should(Succeed()) + }) + + It("Update the VpcAssociationPolicy that set associateWithVpc to true with a SecurityGroupId, expecting the ServiceNetworkVpcAssociation with a security group created", func() { + testFramework.Get(ctx, types.NamespacedName{ + Namespace: vpcAssociationPolicy.Namespace, + Name: vpcAssociationPolicy.Name, + }, vpcAssociationPolicy) + vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(true) + vpcAssociationPolicy.Spec.SecurityGroupIds = []v1alpha1.SecurityGroupId{sgId} + testFramework.ExpectUpdated(ctx, vpcAssociationPolicy) + + Eventually(func(g Gomega) { + associated, snvaId, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork) + g.Expect(err).To(BeNil()) + g.Expect(associated).To(BeTrue()) + output, err := testFramework.LatticeClient.GetServiceNetworkVpcAssociationWithContext(ctx, &vpclattice.GetServiceNetworkVpcAssociationInput{ + ServiceNetworkVpcAssociationIdentifier: &snvaId, + }) + g.Expect(err).To(BeNil()) + g.Expect(output.SecurityGroupIds).To(HaveLen(1)) + g.Expect(*output.SecurityGroupIds[0]).To(Equal(string(sgId))) + }).Should(Succeed()) + }) + + AfterAll(func() { + // Re-create SNVA to clean up the SNVA security group + vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(false) + vpcAssociationPolicy.Spec.SecurityGroupIds = nil + testFramework.ExpectUpdated(ctx, vpcAssociationPolicy) + Eventually(func(g Gomega) { + //Expect no SNVA for test Gateway + associated, _, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork) + g.Expect(associated).To(BeFalse()) + }).Should(Succeed()) + + vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(true) + testFramework.ExpectUpdated(ctx, vpcAssociationPolicy) + Eventually(func(g Gomega) { + // Expect SNVA recreated for testGateway + associated, _, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork) + g.Expect(err).To(BeNil()) + g.Expect(associated).To(BeTrue()) + }).Should(Succeed()) + + // Delete the vpc association policy + testFramework.ExpectDeletedThenNotFound(ctx, vpcAssociationPolicy) + + // Delete the security group + _, err := testFramework.Ec2Client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(string(sgId)), + }) + Expect(err).To(BeNil()) + }) +})