Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create VpcAssociationPolicy e2e test #417

Merged
merged 1 commit into from
Oct 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions test/pkg/test/framework.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"
"time"

"github.com/aws/aws-sdk-go/service/ec2"
"github.com/onsi/gomega/format"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -84,12 +85,15 @@ func addOptionalCRDs(scheme *runtime.Scheme) {
scheme.AddKnownTypes(dnsEndpoint, &endpoint.DNSEndpoint{}, &endpoint.DNSEndpointList{})
metav1.AddToGroupVersion(scheme, dnsEndpoint)

targetGroupPolicy := schema.GroupVersion{
Group: "application-networking.k8s.aws",
awsGatewayControllerCRDGroupVersion := schema.GroupVersion{
Group: v1alpha1.GroupName,
Version: "v1alpha1",
}
scheme.AddKnownTypes(targetGroupPolicy, &v1alpha1.TargetGroupPolicy{}, &v1alpha1.TargetGroupPolicyList{})
metav1.AddToGroupVersion(scheme, targetGroupPolicy)
scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.TargetGroupPolicy{}, &v1alpha1.TargetGroupPolicyList{})
metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion)

scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.VpcAssociationPolicy{}, &v1alpha1.VpcAssociationPolicyList{})
metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion)
}

type Framework struct {
Expand All @@ -100,6 +104,7 @@ type Framework struct {
namespace string
controllerRuntimeConfig *rest.Config
LatticeClient services.Lattice
Ec2Client *ec2.EC2
GrpcurlRunner *v1.Pod
}

Expand All @@ -111,6 +116,7 @@ func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) *
framework := &Framework{
Client: lo.Must(client.New(controllerRuntimeConfig, client.Options{Scheme: testScheme})),
LatticeClient: services.NewDefaultLattice(session.Must(session.NewSession()), config.Region), // region is currently hardcoded
Ec2Client: ec2.New(session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)}))),
GrpcurlRunner: &v1.Pod{},
ctx: ctx,
log: log,
Expand Down Expand Up @@ -362,23 +368,23 @@ func (env *Framework) VerifyTargetGroupNotFound(tg *vpclattice.TargetGroupSummar
}).Should(Succeed())
}

func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpcId string, serviceNetwork *vpclattice.ServiceNetworkSummary) (bool, error) {
func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpcId string, serviceNetwork *vpclattice.ServiceNetworkSummary) (bool, string, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's typically considered an antipattern to return multiple values like this. Shouldn't we return a struct type containing named properties, or error?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you return association id you can omit bool value. Use empty id as false value.

env.log.Infof("IsVpcAssociatedWithServiceNetwork vpcId:%v serviceNetwork: %v \n", vpcId, serviceNetwork)
vpcAssociations, err := env.LatticeClient.ListServiceNetworkVpcAssociationsAsList(ctx, &vpclattice.ListServiceNetworkVpcAssociationsInput{
ServiceNetworkIdentifier: serviceNetwork.Id,
VpcIdentifier: &vpcId,
})
if err != nil {
return false, err
return false, "", err
}
if len(vpcAssociations) != 1 {
return false, fmt.Errorf("Expect to have one VpcServiceNetworkAssociation len(vpcAssociations): %d", len(vpcAssociations))
return false, "", fmt.Errorf("Expect to have one VpcServiceNetworkAssociation len(vpcAssociations): %d", len(vpcAssociations))
}
association := vpcAssociations[0]
if *association.Status != vpclattice.ServiceNetworkVpcAssociationStatusActive {
return false, fmt.Errorf("Current cluster should have one Active status association *association.Status: %s, err: %w", *association.Status, err)
return false, "", fmt.Errorf("Current cluster should have one Active status association *association.Status: %s, err: %w", *association.Status, err)
}
return true, nil
return true, *association.Id, nil
}

func (env *Framework) AreAllLatticeTargetsHealthy(ctx context.Context, tg *vpclattice.TargetGroupSummary) (bool, error) {
Expand Down
13 changes: 8 additions & 5 deletions test/suites/integration/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"flag"
"os"

"github.com/aws/aws-sdk-go/service/vpclattice"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -25,9 +26,10 @@ const (
var testFramework *test.Framework
var ctx context.Context
var testGateway *v1beta1.Gateway
var testServiceNetwork *vpclattice.ServiceNetworkSummary
var _ = BeforeSuite(func() {
vpcid := os.Getenv("CLUSTER_VPC_ID")
if vpcid == "" {
vpcId := os.Getenv("CLUSTER_VPC_ID")
if vpcId == "" {
Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests")
}

Expand All @@ -44,11 +46,12 @@ var _ = BeforeSuite(func() {
testGateway = testFramework.NewGateway("test-gateway", k8snamespace)
testFramework.ExpectCreated(ctx, testGateway)

sn := testFramework.GetServiceNetwork(ctx, testGateway)
testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway)

test.Logger(ctx).Infof("Expecting VPC %s and service network %s association", vpcid, *sn.Id)
test.Logger(ctx).Infof("Expecting VPC %s and service network %s association", vpcId, *testServiceNetwork.Id)
Eventually(func(g Gomega) {
g.Expect(testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcid, sn)).To(BeTrue())
associated, _, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcId, testServiceNetwork)
g.Expect(associated).To(BeTrue())
}).Should(Succeed())
})

Expand Down
144 changes: 144 additions & 0 deletions test/suites/integration/vpc_association_policy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package integration

import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/vpclattice"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/samber/lo"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
gateway_api_v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gateway_api "sigs.k8s.io/gateway-api/apis/v1beta1"

"github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1"
"github.com/aws/aws-application-networking-k8s/test/pkg/test"
)

var _ = Describe("Test vpc association policy", Ordered, func() {
var (
vpcAssociationPolicy *v1alpha1.VpcAssociationPolicy
sgId v1alpha1.SecurityGroupId
)

BeforeAll(func() {
// Create security group
describeVpcOutput, err := testFramework.Ec2Client.DescribeVpcs(&ec2.DescribeVpcsInput{
VpcIds: []*string{aws.String(test.CurrentClusterVpcId)},
})
Expect(err).To(BeNil())
sourceVPC := describeVpcOutput.Vpcs[0]
createSgOutput, err := testFramework.Ec2Client.CreateSecurityGroupWithContext(ctx, &ec2.CreateSecurityGroupInput{
Description: aws.String("k8s-test-lattice-snva-sg"),
GroupName: aws.String("k8s-test-lattice-snva-sg"),
VpcId: aws.String(test.CurrentClusterVpcId),
})
Expect(err).To(BeNil())
sgId = v1alpha1.SecurityGroupId(*createSgOutput.GroupId)

// Create security group inbound rules
_, err = testFramework.Ec2Client.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
GroupId: createSgOutput.GroupId,
IpPermissions: []*ec2.IpPermission{
{ // SG Rule to allow HTTP
IpProtocol: aws.String("tcp"),
IpRanges: []*ec2.IpRange{
{
CidrIp: sourceVPC.CidrBlock,
},
},
FromPort: aws.Int64(80),
ToPort: aws.Int64(80),
},
{ // SG Rule to allow HTTPS
IpProtocol: aws.String("tcp"),
IpRanges: []*ec2.IpRange{
{
CidrIp: sourceVPC.CidrBlock,
},
},
FromPort: aws.Int64(443),
ToPort: aws.Int64(443),
},
},
})
Expect(err).To(BeNil())
})

It("Create a VpcAssociationPolicy that set associateWithVpc to false, expecting do not create ServiceNetworkVpcAssociation", func() {
vpcAssociationPolicy = &v1alpha1.VpcAssociationPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "test-vpc-association-policy",
Namespace: k8snamespace,
},
Spec: v1alpha1.VpcAssociationPolicySpec{
TargetRef: &gateway_api_v1alpha2.PolicyTargetReference{
Group: gateway_api.GroupName,
Kind: "Gateway",
Name: gateway_api_v1alpha2.ObjectName(testGateway.Name),
Namespace: lo.ToPtr(gateway_api_v1alpha2.Namespace(k8snamespace)),
},
AssociateWithVpc: lo.ToPtr(false),
},
}
testFramework.ExpectCreated(ctx, vpcAssociationPolicy)
Eventually(func(g Gomega) {
//Expect no SNVA for testGateway
associated, _, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(err).To(Not(BeNil()))
g.Expect(associated).To(BeFalse())
}).Should(Succeed())
})

It("Update the VpcAssociationPolicy that set associateWithVpc to true with a SecurityGroupId, expecting the ServiceNetworkVpcAssociation with a security group created", func() {
testFramework.Get(ctx, types.NamespacedName{
Namespace: vpcAssociationPolicy.Namespace,
Name: vpcAssociationPolicy.Name,
}, vpcAssociationPolicy)
vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(true)
vpcAssociationPolicy.Spec.SecurityGroupIds = []v1alpha1.SecurityGroupId{sgId}
testFramework.ExpectUpdated(ctx, vpcAssociationPolicy)

Eventually(func(g Gomega) {
associated, snvaId, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(err).To(BeNil())
g.Expect(associated).To(BeTrue())
output, err := testFramework.LatticeClient.GetServiceNetworkVpcAssociationWithContext(ctx, &vpclattice.GetServiceNetworkVpcAssociationInput{
ServiceNetworkVpcAssociationIdentifier: &snvaId,
})
g.Expect(err).To(BeNil())
g.Expect(output.SecurityGroupIds).To(HaveLen(1))
g.Expect(*output.SecurityGroupIds[0]).To(Equal(string(sgId)))
}).Should(Succeed())
})

AfterAll(func() {
// Re-create SNVA to clean up the SNVA security group by setting associateWithVpc to false and then true with no security group
vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(false)
vpcAssociationPolicy.Spec.SecurityGroupIds = nil
testFramework.ExpectUpdated(ctx, vpcAssociationPolicy)
Eventually(func(g Gomega) {
//Expect no SNVA for testGateway
associated, _, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(associated).To(BeFalse())
}).Should(Succeed())
vpcAssociationPolicy.Spec.AssociateWithVpc = lo.ToPtr(true)
testFramework.ExpectUpdated(ctx, vpcAssociationPolicy)
Eventually(func(g Gomega) {
// Expect SNVA re-created for testGateway
associated, _, err := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, test.CurrentClusterVpcId, testServiceNetwork)
g.Expect(err).To(BeNil())
g.Expect(associated).To(BeTrue())
}).Should(Succeed())

// Clean up the vpc association policy
testFramework.ExpectDeletedThenNotFound(ctx, vpcAssociationPolicy)

// Clean up the security group
_, err := testFramework.Ec2Client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
GroupId: aws.String(string(sgId)),
})
Expect(err).To(BeNil())
})
})