Skip to content

Commit

Permalink
Create VpcAssociationPolicy e2e test
Browse files Browse the repository at this point in the history
  • Loading branch information
Zijun Wang committed Sep 29, 2023
1 parent 6cccaa5 commit a29a4b1
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 14 deletions.
24 changes: 15 additions & 9 deletions test/pkg/test/framework.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"
"time"

"github.com/aws/aws-sdk-go/service/ec2"
"github.com/onsi/gomega/format"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -84,12 +85,15 @@ func addOptionalCRDs(scheme *runtime.Scheme) {
scheme.AddKnownTypes(dnsEndpoint, &endpoint.DNSEndpoint{}, &endpoint.DNSEndpointList{})
metav1.AddToGroupVersion(scheme, dnsEndpoint)

targetGroupPolicy := schema.GroupVersion{
Group: "application-networking.k8s.aws",
awsGatewayControllerCRDGroupVersion := schema.GroupVersion{
Group: v1alpha1.GroupName,
Version: "v1alpha1",
}
scheme.AddKnownTypes(targetGroupPolicy, &v1alpha1.TargetGroupPolicy{}, &v1alpha1.TargetGroupPolicyList{})
metav1.AddToGroupVersion(scheme, targetGroupPolicy)
scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.TargetGroupPolicy{}, &v1alpha1.TargetGroupPolicyList{})
metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion)

scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.VpcAssociationPolicy{}, &v1alpha1.VpcAssociationPolicyList{})
metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion)
}

type Framework struct {
Expand All @@ -100,6 +104,7 @@ type Framework struct {
namespace string
controllerRuntimeConfig *rest.Config
LatticeClient services.Lattice
Ec2Client *ec2.EC2
GrpcurlRunner *v1.Pod
}

Expand All @@ -111,6 +116,7 @@ func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) *
framework := &Framework{
Client: lo.Must(client.New(controllerRuntimeConfig, client.Options{Scheme: testScheme})),
LatticeClient: services.NewDefaultLattice(session.Must(session.NewSession()), config.Region), // region is currently hardcoded
Ec2Client: ec2.New(session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)}))),
GrpcurlRunner: &v1.Pod{},
ctx: ctx,
log: log,
Expand Down Expand Up @@ -362,23 +368,23 @@ func (env *Framework) VerifyTargetGroupNotFound(tg *vpclattice.TargetGroupSummar
}).Should(Succeed())
}

func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpcId string, serviceNetwork *vpclattice.ServiceNetworkSummary) (bool, error) {
func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpcId string, serviceNetwork *vpclattice.ServiceNetworkSummary) (bool, string, error) {
env.log.Infof("IsVpcAssociatedWithServiceNetwork vpcId:%v serviceNetwork: %v \n", vpcId, serviceNetwork)
vpcAssociations, err := env.LatticeClient.ListServiceNetworkVpcAssociationsAsList(ctx, &vpclattice.ListServiceNetworkVpcAssociationsInput{
ServiceNetworkIdentifier: serviceNetwork.Id,
VpcIdentifier: &vpcId,
})
if err != nil {
return false, err
return false, "", err
}
if len(vpcAssociations) != 1 {
return false, fmt.Errorf("Expect to have one VpcServiceNetworkAssociation len(vpcAssociations): %d", len(vpcAssociations))
return false, "", fmt.Errorf("Expect to have one VpcServiceNetworkAssociation len(vpcAssociations): %d", len(vpcAssociations))
}
association := vpcAssociations[0]
if *association.Status != vpclattice.ServiceNetworkVpcAssociationStatusActive {
return false, fmt.Errorf("Current cluster should have one Active status association *association.Status: %s, err: %w", *association.Status, err)
return false, "", fmt.Errorf("Current cluster should have one Active status association *association.Status: %s, err: %w", *association.Status, err)
}
return true, nil
return true, *association.Id, nil
}

func (env *Framework) AreAllLatticeTargetsHealthy(ctx context.Context, tg *vpclattice.TargetGroupSummary) (bool, error) {
Expand Down
13 changes: 8 additions & 5 deletions test/suites/integration/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"flag"
"os"

"github.com/aws/aws-sdk-go/service/vpclattice"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -25,9 +26,10 @@ const (
var testFramework *test.Framework
var ctx context.Context
var testGateway *v1beta1.Gateway
var testServiceNetwork *vpclattice.ServiceNetworkSummary
var _ = BeforeSuite(func() {
vpcid := os.Getenv("CLUSTER_VPC_ID")
if vpcid == "" {
vpcId := os.Getenv("CLUSTER_VPC_ID")
if vpcId == "" {
Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests")
}

Expand All @@ -44,11 +46,12 @@ var _ = BeforeSuite(func() {
testGateway = testFramework.NewGateway("test-gateway", k8snamespace)
testFramework.ExpectCreated(ctx, testGateway)

sn := testFramework.GetServiceNetwork(ctx, testGateway)
testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway)

test.Logger(ctx).Infof("Expecting VPC %s and service network %s association", vpcid, *sn.Id)
test.Logger(ctx).Infof("Expecting VPC %s and service network %s association", vpcId, *testServiceNetwork.Id)
Eventually(func(g Gomega) {
g.Expect(testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcid, sn)).To(BeTrue())
associated, _, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcId, testServiceNetwork)
g.Expect(associated).To(BeTrue())
}).Should(Succeed())
})

Expand Down
145 changes: 145 additions & 0 deletions test/suites/integration/vpc_association_policy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
package integration

import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/vpclattice"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/samber/lo"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
gateway_api_v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gateway_api "sigs.k8s.io/gateway-api/apis/v1beta1"

"github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1"
"github.com/aws/aws-application-networking-k8s/test/pkg/test"
)

var _ = Describe("Test vpc association policy", Ordered, Focus, func() {
var (
vpcAssociationPolicy *v1alpha1.VpcAssociationPolicy
sgId v1alpha1.SecurityGroupId
)

BeforeAll(func() {
// Create security group
describeVpcOutput, err := testFramework.Ec2Client.DescribeVpcs(&ec2.DescribeVpcsInput{
VpcIds: []*string{aws.String(test.CurrentClusterVpcId)},
})
Expect(err).To(BeNil())
sourceVPC := describeVpcOutput.Vpcs[0]
createSgOutput, err := testFramework.Ec2Client.CreateSecurityGroupWithContext(ctx, &ec2.CreateSecurityGroupInput{
Description: aws.String("k8s-test-lattice-snva-sg"),
GroupName: aws.String("k8s-test-lattice-snva-sg"),
VpcId: aws.String(test.CurrentClusterVpcId),
})
Expect(err).To(BeNil())
sgId = v1alpha1.SecurityGroupId(*createSgOutput.GroupId)

// Create security group inbound rules
_, err = testFramework.Ec2Client.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
GroupId: createSgOutput.GroupId,
IpPermissions: []*ec2.IpPermission{
{ // SG Rule to allow HTTP
IpProtocol: aws.String("tcp"),
IpRanges: []*ec2.IpRange{
{
CidrIp: sourceVPC.CidrBlock,
},
},
FromPort: aws.Int64(80),
ToPort: aws.Int64(80),
},
{ // SG Rule to allow HTTPS
IpProtocol: aws.String("tcp"),
IpRanges: []*ec2.IpRange{
{
CidrIp: sourceVPC.CidrBlock,
},
},
FromPort: aws.Int64(443),
ToPort: aws.Int64(443),
},
},
})
Expect(err).To(BeNil())
})

It("Create a VpcAssociationPolicy that set associateWithVpc to false, expecting do not create ServiceNetworkVpcAssociation", func() {
vpcAssociationPolicy = &v1alpha1.VpcAssociationPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "test-vpc-association-policy",
Namespace: k8snamespace,
},
Spec: v1alpha1.VpcAssociationPolicySpec{
TargetRef: &gateway_api_v1alpha2.PolicyTargetReference{
Group: gateway_api.GroupName,
Kind: "Gateway",
Name: gateway_api_v1alpha2.ObjectName(testGateway.Name),
Namespace: lo.ToPtr(gateway_api_v1alpha2.Namespace(k8snamespace)),
},
AssociateWithVpc: lo.ToPtr(false),
},
}
testFramework.ExpectCreated(ctx, vpcAssociationPolicy)
Eventually(func(g Gomega) {
//Expect no SNVA for testGateway
associated, _, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(err).To(Not(BeNil()))
g.Expect(associated).To(BeFalse())
}).Should(Succeed())
})

It("Update the VpcAssociationPolicy that set associateWithVpc to true with a SecurityGroupId, expecting the ServiceNetworkVpcAssociation with a security group created", func() {
testFramework.Get(ctx, types.NamespacedName{
Namespace: vpcAssociationPolicy.Namespace,
Name: vpcAssociationPolicy.Name,
}, vpcAssociationPolicy)
vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(true)
vpcAssociationPolicy.Spec.SecurityGroupIds = []v1alpha1.SecurityGroupId{sgId}
testFramework.ExpectUpdated(ctx, vpcAssociationPolicy)

Eventually(func(g Gomega) {
associated, snvaId, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(err).To(BeNil())
g.Expect(associated).To(BeTrue())
output, err := testFramework.LatticeClient.GetServiceNetworkVpcAssociationWithContext(ctx, &vpclattice.GetServiceNetworkVpcAssociationInput{
ServiceNetworkVpcAssociationIdentifier: &snvaId,
})
g.Expect(err).To(BeNil())
g.Expect(output.SecurityGroupIds).To(HaveLen(1))
g.Expect(*output.SecurityGroupIds[0]).To(Equal(string(sgId)))
}).Should(Succeed())
})

AfterAll(func() {
// Re-create SNVA to clean up the SNVA security group
vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(false)
vpcAssociationPolicy.Spec.SecurityGroupIds = nil
testFramework.ExpectUpdated(ctx, vpcAssociationPolicy)
Eventually(func(g Gomega) {
//Expect no SNVA for testGateway
associated, _, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(associated).To(BeFalse())
}).Should(Succeed())

vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(true)
testFramework.ExpectUpdated(ctx, vpcAssociationPolicy)
Eventually(func(g Gomega) {
// Expect SNVA recreated for testGateway
associated, _, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(err).To(BeNil())
g.Expect(associated).To(BeTrue())
}).Should(Succeed())

// Delete the vpc association policy
testFramework.ExpectDeletedThenNotFound(ctx, vpcAssociationPolicy)

// Delete the security group
_, err := testFramework.Ec2Client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
GroupId: aws.String(string(sgId)),
})
Expect(err).To(BeNil())
})
})

0 comments on commit a29a4b1

Please sign in to comment.