diff --git a/test/go.mod b/test/go.mod index 41bc514b..7e3d960a 100644 --- a/test/go.mod +++ b/test/go.mod @@ -13,6 +13,7 @@ require ( github.com/onsi/gomega v1.27.4 github.com/samber/lo v1.37.0 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 k8s.io/api v0.26.2 k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.2 diff --git a/test/pkg/test/framework.go b/test/pkg/test/framework.go index fbed1778..6b3a5bd1 100644 --- a/test/pkg/test/framework.go +++ b/test/pkg/test/framework.go @@ -34,8 +34,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - gateway_api_v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" - gateway_api_v1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" mcs_api "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" "github.com/aws/aws-application-networking-k8s/controllers" @@ -59,10 +59,10 @@ var ( testScheme = runtime.NewScheme() CurrentClusterVpcId = os.Getenv("CLUSTER_VPC_ID") TestObjects = []TestObject{ - {&gateway_api_v1beta1.HTTPRoute{}, &gateway_api_v1beta1.HTTPRouteList{}}, + {&gwv1beta1.HTTPRoute{}, &gwv1beta1.HTTPRouteList{}}, {&mcs_api.ServiceExport{}, &mcs_api.ServiceExportList{}}, {&mcs_api.ServiceImport{}, &mcs_api.ServiceImportList{}}, - {&gateway_api_v1beta1.Gateway{}, &gateway_api_v1beta1.GatewayList{}}, + {&gwv1beta1.Gateway{}, &gwv1beta1.GatewayList{}}, {&appsv1.Deployment{}, &appsv1.DeploymentList{}}, {&v1.Service{}, &v1.ServiceList{}}, } @@ -71,8 +71,8 @@ var ( func init() { format.MaxLength = 0 utilruntime.Must(clientgoscheme.AddToScheme(testScheme)) - utilruntime.Must(gateway_api_v1alpha2.AddToScheme(testScheme)) - utilruntime.Must(gateway_api_v1beta1.AddToScheme(testScheme)) + utilruntime.Must(gwv1alpha2.AddToScheme(testScheme)) + utilruntime.Must(gwv1beta1.AddToScheme(testScheme)) utilruntime.Must(mcs_api.AddToScheme(testScheme)) addOptionalCRDs(testScheme) } @@ -99,10 +99,10 @@ func addOptionalCRDs(scheme *runtime.Scheme) { type Framework struct { client.Client ctx context.Context - log gwlog.Logger k8sScheme *runtime.Scheme namespace string controllerRuntimeConfig *rest.Config + Log gwlog.Logger LatticeClient services.Lattice Ec2Client *ec2.EC2 GrpcurlRunner *v1.Pod @@ -119,7 +119,7 @@ func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) * Ec2Client: ec2.New(session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)}))), GrpcurlRunner: &v1.Pod{}, ctx: ctx, - log: log, + Log: log, k8sScheme: testScheme, namespace: testNamespace, controllerRuntimeConfig: controllerRuntimeConfig, @@ -130,7 +130,7 @@ func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) * } func (env *Framework) ExpectToBeClean(ctx context.Context) { - env.log.Info("Expecting the test environment to be clean") + env.Log.Info("Expecting the test environment to be clean") // Kubernetes API Objects parallel.ForEach(TestObjects, func(testObject TestObject, _ int) { defer GinkgoRecover() @@ -140,17 +140,17 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { retrievedServiceNetworkVpcAssociations, _ := env.LatticeClient.ListServiceNetworkVpcAssociationsAsList(ctx, &vpclattice.ListServiceNetworkVpcAssociationsInput{ VpcIdentifier: aws.String(CurrentClusterVpcId), }) - env.log.Infof("Expect VPC used by current cluster has no ServiceNetworkVPCAssociation, if it does you should manually delete it") + env.Log.Infof("Expect VPC used by current cluster has no ServiceNetworkVPCAssociation, if it does you should manually delete it") Expect(len(retrievedServiceNetworkVpcAssociations)).To(Equal(0)) Eventually(func(g Gomega) { retrievedServiceNetworks, _ := env.LatticeClient.ListServiceNetworksAsList(ctx, &vpclattice.ListServiceNetworksInput{}) for _, sn := range retrievedServiceNetworks { - env.log.Infof("Found service network, checking if created by current EKS Cluster: %v", sn) + env.Log.Infof("Found service network, checking if created by current EKS Cluster: %v", sn) retrievedTags, err := env.LatticeClient.ListTagsForResourceWithContext(ctx, &vpclattice.ListTagsForResourceInput{ ResourceArn: sn.Arn, }) if err == nil { // for err != nil, it is possible that this service network own by other account, and it is shared to current account by RAM - env.log.Infof("Found Tags for serviceNetwork %v tags: %v", *sn.Name, retrievedTags) + env.Log.Infof("Found Tags for serviceNetwork %v tags: %v", *sn.Name, retrievedTags) value, ok := retrievedTags.Tags[lattice.K8SServiceNetworkOwnedByVPC] if ok { @@ -161,12 +161,12 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { retrievedServices, _ := env.LatticeClient.ListServicesAsList(ctx, &vpclattice.ListServicesInput{}) for _, service := range retrievedServices { - env.log.Infof("Found service, checking if created by current EKS Cluster: %v", service) + env.Log.Infof("Found service, checking if created by current EKS Cluster: %v", service) retrievedTags, err := env.LatticeClient.ListTagsForResourceWithContext(ctx, &vpclattice.ListTagsForResourceInput{ ResourceArn: service.Arn, }) if err == nil { // for err != nil, it is possible that this service own by other account, and it is shared to current account by RAM - env.log.Infof("Found Tags for service %v tags: %v", *service.Name, retrievedTags) + env.Log.Infof("Found Tags for service %v tags: %v", *service.Name, retrievedTags) value, ok := retrievedTags.Tags[lattice.K8SServiceOwnedByVPC] if ok { g.Expect(*value).To(Not(Equal(CurrentClusterVpcId))) @@ -176,9 +176,9 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { retrievedTargetGroups, _ := env.LatticeClient.ListTargetGroupsAsList(ctx, &vpclattice.ListTargetGroupsInput{}) for _, tg := range retrievedTargetGroups { - env.log.Infof("Found TargetGroup: %s, checking if created by current EKS Cluster", *tg.Id) + env.Log.Infof("Found TargetGroup: %s, checking if created by current EKS Cluster", *tg.Id) if tg.VpcIdentifier != nil && CurrentClusterVpcId != *tg.VpcIdentifier { - env.log.Infof("Target group VPC Id: %s, does not match current EKS Cluster VPC Id: %s", *tg.VpcIdentifier, CurrentClusterVpcId) + env.Log.Infof("Target group VPC Id: %s, does not match current EKS Cluster VPC Id: %s", *tg.VpcIdentifier, CurrentClusterVpcId) //This tg is not created by current EKS Cluster, skip it continue } @@ -186,10 +186,10 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { ResourceArn: tg.Arn, }) if err == nil { - env.log.Infof("Found Tags for tg %v tags: %v", *tg.Name, retrievedTags) + env.Log.Infof("Found Tags for tg %v tags: %v", *tg.Name, retrievedTags) tagValue, ok := retrievedTags.Tags[lattice.K8SParentRefTypeKey] if ok && *tagValue == lattice.K8SServiceExportType { - env.log.Infof("TargetGroup: %s was created by k8s controller, by a ServiceExport", *tg.Id) + env.Log.Infof("TargetGroup: %s was created by k8s controller, by a ServiceExport", *tg.Id) //This tg is created by k8s controller, by a ServiceExport, //ServiceExport still have a known targetGroup leaking issue, //so we temporarily skip to verify whether ServiceExport created TargetGroup is deleted or not @@ -202,14 +202,14 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { func (env *Framework) ExpectCreated(ctx context.Context, objects ...client.Object) { for _, object := range objects { - env.log.Infof("Creating %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) + env.Log.Infof("Creating %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) Expect(env.Create(ctx, object)).WithOffset(1).To(Succeed()) } } func (env *Framework) ExpectUpdated(ctx context.Context, objects ...client.Object) { for _, object := range objects { - env.log.Infof("Updating %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) + env.Log.Infof("Updating %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) Expect(env.Update(ctx, object)).WithOffset(1).To(Succeed()) } } @@ -221,7 +221,7 @@ func (env *Framework) ExpectDeletedThenNotFound(ctx context.Context, objects ... func (env *Framework) ExpectDeleted(ctx context.Context, objects ...client.Object) { for _, object := range objects { - env.log.Infof("Deleting %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) + env.Log.Infof("Deleting %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) err := env.Delete(ctx, object) if err != nil { // not found is probably OK - means it was deleted elsewhere @@ -240,7 +240,7 @@ func (env *Framework) EventuallyExpectNotFound(ctx context.Context, objects ...c Eventually(func(g Gomega) { for _, object := range objects { if object != nil { - env.log.Infof("Checking whether %s %s/%s is not found", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) + env.Log.Infof("Checking whether %s %s/%s is not found", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) g.Expect(errors.IsNotFound(env.Get(ctx, client.ObjectKeyFromObject(object), object))).To(BeTrue()) } } @@ -256,7 +256,7 @@ func (env *Framework) EventuallyExpectNoneFound(ctx context.Context, objectList }).WithOffset(1).Should(Succeed()) } -func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *gateway_api_v1beta1.Gateway) *vpclattice.ServiceNetworkSummary { +func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *gwv1beta1.Gateway) *vpclattice.ServiceNetworkSummary { var found *vpclattice.ServiceNetworkSummary Eventually(func(g Gomega) { listServiceNetworksOutput, err := env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{}) @@ -274,7 +274,7 @@ func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *gateway_ap func (env *Framework) GetVpcLatticeService(ctx context.Context, route core.Route) *vpclattice.ServiceSummary { var found *vpclattice.ServiceSummary - rnProvider := controllers.RouteLSNProvider{route} + rnProvider := controllers.RouteLSNProvider{Route: route} Eventually(func(g Gomega) { svc, err := env.LatticeClient.FindService(ctx, &rnProvider) @@ -334,15 +334,15 @@ func (env *Framework) GetAllTargets(ctx context.Context, targetGroup *vpclattice } func GetTargets(targetGroup *vpclattice.TargetGroupSummary, deployment *appsv1.Deployment, env *Framework, ctx context.Context) ([]string, []*vpclattice.TargetSummary) { - env.log.Infoln("Trying to retrieve registered targets for targetGroup", targetGroup.Name) - env.log.Infoln("deployment.Spec.Selector.MatchLabels:", deployment.Spec.Selector.MatchLabels) + env.Log.Infoln("Trying to retrieve registered targets for targetGroup", targetGroup.Name) + env.Log.Infoln("deployment.Spec.Selector.MatchLabels:", deployment.Spec.Selector.MatchLabels) podList := &v1.PodList{} expectedMatchingLabels := make(map[string]string, len(deployment.Spec.Selector.MatchLabels)) for k, v := range deployment.Spec.Selector.MatchLabels { expectedMatchingLabels[k] = v } expectedMatchingLabels[DiscoveryLabel] = "true" - env.log.Infoln("Expected matching labels:", expectedMatchingLabels) + env.Log.Infoln("Expected matching labels:", expectedMatchingLabels) Expect(env.List(ctx, podList, client.MatchingLabels(expectedMatchingLabels))).To(Succeed()) Expect(podList.Items).To(HaveLen(int(*deployment.Spec.Replicas))) retrievedTargets, err := env.LatticeClient.ListTargetsAsList(ctx, &vpclattice.ListTargetsInput{TargetGroupIdentifier: targetGroup.Id}) @@ -369,7 +369,6 @@ func (env *Framework) VerifyTargetGroupNotFound(tg *vpclattice.TargetGroupSummar } func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpcId string, serviceNetwork *vpclattice.ServiceNetworkSummary) (bool, string, error) { - env.log.Infof("IsVpcAssociatedWithServiceNetwork vpcId:%v serviceNetwork: %v \n", vpcId, serviceNetwork) vpcAssociations, err := env.LatticeClient.ListServiceNetworkVpcAssociationsAsList(ctx, &vpclattice.ListServiceNetworkVpcAssociationsInput{ ServiceNetworkIdentifier: serviceNetwork.Id, VpcIdentifier: &vpcId, @@ -388,13 +387,13 @@ func (env *Framework) IsVpcAssociatedWithServiceNetwork(ctx context.Context, vpc } func (env *Framework) AreAllLatticeTargetsHealthy(ctx context.Context, tg *vpclattice.TargetGroupSummary) (bool, error) { - env.log.Infof("Checking whether AreAllLatticeTargetsHealthy for targetGroup: %v", tg) + env.Log.Infof("Checking whether AreAllLatticeTargetsHealthy for targetGroup: %v", tg) targets, err := env.LatticeClient.ListTargetsAsList(ctx, &vpclattice.ListTargetsInput{TargetGroupIdentifier: tg.Id}) if err != nil { return false, err } for _, target := range targets { - env.log.Infof("Checking target: %v", target) + env.Log.Infof("Checking target: %v", target) if *target.Status != vpclattice.TargetStatusHealthy { return false, nil } @@ -452,8 +451,8 @@ func (env *Framework) GetLatticeServiceHttpsListenerNonDefaultRules(ctx context. } func (env *Framework) GetVpcLatticeServiceDns(httpRouteName string, httpRouteNamespace string) string { - env.log.Infoln("GetVpcLatticeServiceDns: ", httpRouteName, httpRouteNamespace) - httproute := gateway_api_v1beta1.HTTPRoute{} + env.Log.Infoln("GetVpcLatticeServiceDns: ", httpRouteName, httpRouteNamespace) + httproute := gwv1beta1.HTTPRoute{} env.Get(env.ctx, types.NamespacedName{Name: httpRouteName, Namespace: httpRouteNamespace}, &httproute) vpcLatticeServiceDns := httproute.Annotations[controllers.LatticeAssignedDomainName] return vpcLatticeServiceDns @@ -472,7 +471,7 @@ type RunGrpcurlCmdOptions struct { // https://github.com/fullstorydev/grpcurl // https://gallery.ecr.aws/a0j4q9e4/grpcurl-runner func (env *Framework) RunGrpcurlCmd(opts RunGrpcurlCmdOptions) (string, string, error) { - env.log.Infoln("RunGrpcurlCmd") + env.Log.Infoln("RunGrpcurlCmd") Expect(env.GrpcurlRunner).To(Not(BeNil())) tlsOption := "" @@ -503,9 +502,7 @@ func (env *Framework) RunGrpcurlCmd(opts RunGrpcurlCmdOptions) (string, string, opts.Service, opts.Method) - stdoutStr, stderrStr, err := env.PodExec(env.GrpcurlRunner.Namespace, env.GrpcurlRunner.Name, cmd, false) - return stdoutStr, stderrStr, err - + return env.PodExec(*env.GrpcurlRunner, cmd) } func (env *Framework) SleepForRouteDeletion() { diff --git a/test/pkg/test/pod_manager.go b/test/pkg/test/pod_manager.go index 37da4817..475f3a2c 100644 --- a/test/pkg/test/pod_manager.go +++ b/test/pkg/test/pod_manager.go @@ -5,7 +5,7 @@ import ( "net/http" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -18,9 +18,8 @@ import ( ) // https://github.com/aws/amazon-vpc-cni-k8s/blob/7eeb2a9ab437887f77de30a5eab20bb42742df06/test/framework/resources/k8s/resources/pod.go#L188 -func (env *Framework) PodExec(namespace string, podName string, cmd string, printOutput bool) (string, string, error) { - env.log.Infof("PodExec() [namespace: %v] [podName: %v] [command: %v] \n", namespace, podName, cmd) - restClient, err := env.getRestClientForPod(namespace, podName) +func (env *Framework) PodExec(pod corev1.Pod, cmd string) (string, string, error) { + restClient, err := env.getRestClientForPod(pod.Namespace, pod.Name) if err != nil { return "", "", err } @@ -29,7 +28,7 @@ func (env *Framework) PodExec(namespace string, podName string, cmd string, prin "-c", cmd, } - execOptions := &v1.PodExecOptions{ + execOptions := &corev1.PodExecOptions{ Stdout: true, Stderr: true, Command: command, @@ -38,8 +37,8 @@ func (env *Framework) PodExec(namespace string, podName string, cmd string, prin restClient.Get() req := restClient.Post(). Resource("pods"). - Name(podName). - Namespace(namespace). + Name(pod.Name). + Namespace(pod.Namespace). SubResource("exec"). VersionedParams(execOptions, runtime.NewParameterCodec(scheme.Scheme)) @@ -53,20 +52,15 @@ func (env *Framework) PodExec(namespace string, podName string, cmd string, prin Stdout: &stdout, Stderr: &stderr, }) - stdoutStr := stdout.String() - stderrStr := stderr.String() - if printOutput { - env.log.Infow("pod exec output", "stdout", stdoutStr, "stderr", stderrStr) - } - return stdoutStr, stderrStr, err + return stdout.String(), stderr.String(), err } -func (env *Framework) GetPodsByDeploymentName(deploymentName string, deploymentNamespce string) []v1.Pod { +func (env *Framework) GetPodsByDeploymentName(deploymentName string, deploymentNamespce string) []corev1.Pod { deployment := appsv1.Deployment{} env.Get(env.ctx, types.NamespacedName{Name: deploymentName, Namespace: deploymentNamespce}, &deployment) - pods := &v1.PodList{} - env.log.Infoln("deployment.Spec.Selector.MatchLabels:", deployment.Spec.Selector.MatchLabels) + pods := &corev1.PodList{} + env.Log.Infoln("deployment.Spec.Selector.MatchLabels:", deployment.Spec.Selector.MatchLabels) env.List(env.ctx, pods, client.MatchingLabelsSelector{ Selector: labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), }) @@ -74,7 +68,7 @@ func (env *Framework) GetPodsByDeploymentName(deploymentName string, deploymentN } func (env *Framework) getRestClientForPod(namespace string, name string) (rest.Interface, error) { - pod := &v1.Pod{} + pod := &corev1.Pod{} err := env.Get(env.ctx, types.NamespacedName{ Namespace: namespace, Name: name, diff --git a/test/suites/integration/byoc_test.go b/test/suites/integration/byoc_test.go new file mode 100644 index 00000000..1db03616 --- /dev/null +++ b/test/suites/integration/byoc_test.go @@ -0,0 +1,417 @@ +package integration + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" + + "github.com/aws/aws-application-networking-k8s/pkg/config" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/acm" + "github.com/aws/aws-sdk-go/service/route53" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/exp/slices" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +var _ = Describe("Bring your own certificate (BYOC)", Ordered, func() { + + const ( + hostedZoneName = "e2e-test.com" + certDnsName = "*." + hostedZoneName + cname = "byoc." + hostedZoneName + ) + + var ( + log = testFramework.Log.Named("byoc") + sess = session.New() + awsCfg = aws.NewConfig().WithRegion(config.Region) + acmClient = acm.New(sess, awsCfg) + r53Client = route53.New(sess) + + certArn string + hostedZoneId string + latticeSvcDns string + err error + deployment *appsv1.Deployment + service *corev1.Service + httpRoute *gwv1beta1.HTTPRoute + ) + + BeforeAll(func() { + log := log.Named("before-all") + // create and deploy certificate to acm + certArn, err = createCertIfNotExists(acmClient, certDnsName) + Expect(err).To(BeNil()) + log.Infof("created certificate: %s", certArn) + + // add new certificate to gateway spec + addGatewayBYOCListener(certArn) + log.Infof("added listener with cert to gateway") + + // create and deploy service for traffic test + deployment, service = testFramework.NewHttpApp(test.HTTPAppOptions{ + Name: "byoc-app", + Namespace: k8snamespace, + }) + + // create http route with custom certificate setting + httpRoute = testFramework.NewHttpRoute(testGateway, service, "Service") + httpRoute.Spec.Hostnames = []gwv1beta1.Hostname{gwv1beta1.Hostname(cname)} + sectionName := gwv1beta1.SectionName("byoc") + httpRoute.Spec.ParentRefs[0].SectionName = §ionName + testFramework.ExpectCreated(context.TODO(), deployment, service, httpRoute) + + // get lattice service dns name for route53 cname + svc := testFramework.GetVpcLatticeService(context.TODO(), core.NewHTTPRoute(*httpRoute)) + latticeSvcDns = *svc.DnsEntry.DomainName + log.Infof("depoloyed lattice service, dns name: %s", latticeSvcDns) + + // create route 53 hosted zone and cname + hz, err := createHostedZoneIfNotExists(r53Client, hostedZoneName) + Expect(err).To(BeNil()) + hostedZoneId = *hz.Id + log.Infof("created route53 hosted zone, id: %s", hostedZoneId) + err = createCnameIfNotExists(r53Client, hostedZoneId, cname, latticeSvcDns) + Expect(err).To(BeNil()) + log.Infof("created cname for lattice service, cname: %s, value: %s", cname, latticeSvcDns) + }) + + It("same pod https traffic test", func() { + log := log.Named("traffic test") + pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) + pod := pods[0] + Eventually(func(g Gomega) { + cmd := fmt.Sprintf("curl -v -k https://%s/", cname) + log.Infof("calling lattice service, cmd=%s, pod=%s/%s", cmd, pod.Namespace, pod.Name) + stdout, stderr, err := testFramework.PodExec(pod, cmd) + g.Expect(err).To(BeNil()) + g.Expect(stdout).To(ContainSubstring("byoc-app handler pod")) + g.Expect(stderr).To(ContainSubstring("issuer: O=byoc-e2e-test")) + }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) + }) + + AfterAll(func() { + log := log.Named("after-all") + err = deleteHostedZoneRecords(r53Client, hostedZoneId) + Expect(err).To(BeNil()) + log.Infof("deleted hosted zone records for, id: %s", hostedZoneId) + + err = deleteHostedZone(r53Client, hostedZoneId) + Expect(err).To(BeNil()) + log.Infof("deleted route53 hosted zone, id: %s", hostedZoneId) + + testFramework.ExpectDeleted(context.TODO(), httpRoute, service, deployment) + testFramework.SleepForRouteDeletion() + testFramework.ExpectDeletedThenNotFound(context.TODO(), httpRoute, service, deployment) + + removeGatewayBYOCListener() + log.Infof("removed listener with custom cert from gateway") + + err = deleteCert(acmClient, certArn) + Expect(err).To(BeNil()) + log.Infof("removed custom cert from acm, arn: %s", certArn) + }) +}) + +func createCertIfNotExists(client *acm.ACM, dnsName string) (string, error) { + arn, err := findCert(client, dnsName) + if err != nil { + return "", err + } + if arn != "" { + return arn, nil + } + arn, err = createCert(client, dnsName) + if err != nil { + return "", err + } + return arn, nil +} + +func createCert(client *acm.ACM, dnsName string) (string, error) { + cert, priv, err := genCert(dnsName) + if err != nil { + return "", err + } + arn, err := uploadCert(client, cert, priv) + if err != nil { + return "", err + } + return arn, nil +} +func genCert(dnsName string) ([]byte, []byte, error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + template := x509.Certificate{ + SerialNumber: big.NewInt(time.Now().Unix()), + Subject: pkix.Name{ + Organization: []string{"byoc-e2e-test"}, + }, + DNSNames: []string{dnsName}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 7), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + derCert, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, nil, err + } + pemCert, err := derToPem(derCert, "CERTIFICATE") + if err != nil { + return nil, nil, err + } + derPriv, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + return nil, nil, err + } + pemPriv, err := derToPem(derPriv, "PRIVATE KEY") + if err != nil { + return nil, nil, err + } + return pemCert, pemPriv, nil +} + +func derToPem(der []byte, block string) ([]byte, error) { + var buf bytes.Buffer + err := pem.Encode(&buf, &pem.Block{ + Type: block, + Bytes: der, + }) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func findCert(client *acm.ACM, dnsName string) (string, error) { + out, err := client.ListCertificates(&acm.ListCertificatesInput{ + CertificateStatuses: []*string{aws.String(acm.CertificateStatusIssued)}, + }) + if err != nil { + return "", err + } + for _, cert := range out.CertificateSummaryList { + if *cert.DomainName == dnsName { + return *cert.CertificateArn, nil + } + } + return "", nil +} + +func uploadCert(client *acm.ACM, cert []byte, priv []byte) (string, error) { + out, err := client.ImportCertificate(&acm.ImportCertificateInput{ + Certificate: cert, + PrivateKey: priv, + }) + if err != nil { + return "", err + } + return *out.CertificateArn, nil +} + +func deleteCert(client *acm.ACM, arn string) error { + _, err := client.DeleteCertificate(&acm.DeleteCertificateInput{ + CertificateArn: &arn, + }) + return err +} + +func addGatewayBYOCListener(certArn string) { + gw := &gwv1beta1.Gateway{} + testFramework.Get(context.TODO(), types.NamespacedName{ + Namespace: testGateway.Namespace, + Name: testGateway.Name, + }, gw) + tlsMode := gwv1beta1.TLSModeTerminate + byocListener := gwv1beta1.Listener{ + Name: "byoc", + Port: 443, + Protocol: gwv1beta1.HTTPSProtocolType, + TLS: &gwv1beta1.GatewayTLSConfig{ + Mode: &tlsMode, + Options: map[gwv1beta1.AnnotationKey]gwv1beta1.AnnotationValue{ + "application-networking.k8s.aws/certificate-arn": gwv1beta1.AnnotationValue(certArn), + }, + }, + } + gw.Spec.Listeners = append(gw.Spec.Listeners, byocListener) + testFramework.ExpectUpdated(context.TODO(), gw) +} + +func removeGatewayBYOCListener() { + gw := &gwv1beta1.Gateway{} + testFramework.Get(context.TODO(), types.NamespacedName{ + Namespace: testGateway.Namespace, + Name: testGateway.Name, + }, gw) + gw.Spec.Listeners = slices.DeleteFunc(gw.Spec.Listeners, + func(l gwv1beta1.Listener) bool { + return l.Name == "byoc" + }) + testFramework.ExpectUpdated(context.TODO(), gw) +} + +func createHostedZoneIfNotExists(client *route53.Route53, hostedZoneName string) (*route53.HostedZone, error) { + hostedZone, err := findHostedZone(client, hostedZoneName) + if err != nil { + return nil, err + } + if hostedZone != nil { + return hostedZone, nil + } + hostedZone, err = createHostedZone(client, hostedZoneName) + if err != nil { + return nil, err + } + return hostedZone, nil +} + +func findHostedZone(client *route53.Route53, name string) (*route53.HostedZone, error) { + out, err := client.ListHostedZonesByName(&route53.ListHostedZonesByNameInput{DNSName: &name}) + if err != nil { + return nil, err + } + if len(out.HostedZones) == 0 { + return nil, nil + } + return out.HostedZones[0], nil +} + +func createHostedZone(client *route53.Route53, name string) (*route53.HostedZone, error) { + out, err := client.CreateHostedZone(&route53.CreateHostedZoneInput{ + CallerReference: aws.String(fmt.Sprintf("%s-%d", name, time.Now().Unix())), + HostedZoneConfig: &route53.HostedZoneConfig{ + Comment: aws.String("eks byoc test"), + PrivateZone: aws.Bool(true), + }, + VPC: &route53.VPC{VPCId: &config.VpcID, VPCRegion: &config.Region}, + Name: &name, + }) + if err != nil { + return nil, err + } + err = client.WaitUntilResourceRecordSetsChanged(&route53.GetChangeInput{Id: out.ChangeInfo.Id}) + if err != nil { + return nil, err + } + return out.HostedZone, nil +} + +func createCnameIfNotExists(client *route53.Route53, hostedZoneId, cname, cvalue string) error { + exists, err := cnameExists(client, hostedZoneId, cname) + if err != nil { + return err + } + if exists { + return nil + } + err = createCname(client, hostedZoneId, cname, cvalue) + if err != nil { + return err + } + return nil +} + +func createCname(client *route53.Route53, hostedZoneId, cname string, cvalue string) error { + out, err := client.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{ + ChangeBatch: &route53.ChangeBatch{ + Changes: []*route53.Change{ + { + Action: aws.String(route53.ChangeActionCreate), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: &cname, + ResourceRecords: []*route53.ResourceRecord{{Value: &cvalue}}, + TTL: aws.Int64(300), + Type: aws.String(route53.RRTypeCname), + }, + }, + }, + Comment: aws.String("create cname for byoc"), + }, + HostedZoneId: &hostedZoneId, + }) + if err != nil { + return err + } + err = client.WaitUntilResourceRecordSetsChanged(&route53.GetChangeInput{Id: out.ChangeInfo.Id}) + if err != nil { + return err + } + return nil +} + +func cnameExists(client *route53.Route53, hostedZoneId, cname string) (bool, error) { + rrs, err := client.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{ + HostedZoneId: &hostedZoneId, + }) + if err != nil { + return false, err + } + for _, rec := range rrs.ResourceRecordSets { + if *rec.Name == cname+"." && *rec.Type == route53.RRTypeCname { + return true, nil + } + } + return false, nil +} + +func deleteHostedZoneRecords(client *route53.Route53, hostedZoneId string) error { + rrs, err := client.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{ + HostedZoneId: &hostedZoneId, + }) + changes := []*route53.Change{} + for _, rec := range rrs.ResourceRecordSets { + if *rec.Type == route53.RRTypeNs || *rec.Type == route53.RRTypeSoa { + continue + } + changes = append(changes, &route53.Change{ + Action: aws.String(route53.ChangeActionDelete), + ResourceRecordSet: rec, + }) + } + out, err := client.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{ + ChangeBatch: &route53.ChangeBatch{ + Changes: changes, + Comment: aws.String("cleanup byoc test"), + }, + HostedZoneId: &hostedZoneId, + }) + err = client.WaitUntilResourceRecordSetsChanged(&route53.GetChangeInput{Id: out.ChangeInfo.Id}) + if err != nil { + return err + } + return nil +} + +func deleteHostedZone(client *route53.Route53, hostedZoneId string) error { + out, err := client.DeleteHostedZone(&route53.DeleteHostedZoneInput{ + Id: &hostedZoneId, + }) + if err != nil { + return err + } + err = client.WaitUntilResourceRecordSetsChanged(&route53.GetChangeInput{Id: out.ChangeInfo.Id}) + if err != nil { + return err + } + return nil +} diff --git a/test/suites/integration/defined_target_ports_test.go b/test/suites/integration/defined_target_ports_test.go index 77eb130a..63a0f83e 100644 --- a/test/suites/integration/defined_target_ports_test.go +++ b/test/suites/integration/defined_target_ports_test.go @@ -1,9 +1,10 @@ package integration import ( - "github.com/aws/aws-application-networking-k8s/controllers" "os" + "github.com/aws/aws-application-networking-k8s/controllers" + "github.com/aws/aws-sdk-go/service/vpclattice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -70,7 +71,7 @@ var _ = Describe("Defined Target Ports", func() { // Verify VPC Lattice Service exists route, _ := core.NewRoute(httpRoute) vpcLatticeService = testFramework.GetVpcLatticeService(ctx, route) - rnp := controllers.RouteLSNProvider{route} + rnp := controllers.RouteLSNProvider{Route: route} Expect(*vpcLatticeService.DnsEntry).To(ContainSubstring(rnp.LatticeServiceName())) performVerification(service, deployment, definedPorts) diff --git a/test/suites/integration/httproute_header_match_test.go b/test/suites/integration/httproute_header_match_test.go index afd7f2bf..3a880ac1 100644 --- a/test/suites/integration/httproute_header_match_test.go +++ b/test/suites/integration/httproute_header_match_test.go @@ -78,7 +78,7 @@ var _ = Describe("HTTPRoute header matches", func() { testFramework.Get(ctx, types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, deployment) pods := testFramework.GetPodsByDeploymentName(deployment.Name, deployment.Namespace) Expect(len(pods)).To(BeEquivalentTo(1)) - log.Println("pods[0].Name:", pods[0].Name) + pod := pods[0] // after rules in place, it can take some time for listener rules to fully propagate log.Println("Verifying traffic") @@ -86,7 +86,7 @@ var _ = Describe("HTTPRoute header matches", func() { // check correct headers Eventually(func(g Gomega) { cmd := fmt.Sprintf("curl %s -H \"my-header-name1: my-header-value1\" -H \"my-header-name2: my-header-value2\"", dnsName) - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd, true) + stdout, _, err := testFramework.PodExec(pod, cmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("test-v3 handler pod")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) @@ -94,7 +94,7 @@ var _ = Describe("HTTPRoute header matches", func() { // check incorrect headers Eventually(func(g Gomega) { invalidCmd := fmt.Sprintf("curl %s -H \"my-header-name1: my-header-value1\" -H \"my-header-name2: value2-invalid\"", dnsName) - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, invalidCmd, true) + stdout, _, err := testFramework.PodExec(pod, invalidCmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("Not Found")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) diff --git a/test/suites/integration/httproute_method_match_test.go b/test/suites/integration/httproute_method_match_test.go index 4336de0c..0544c8d5 100644 --- a/test/suites/integration/httproute_method_match_test.go +++ b/test/suites/integration/httproute_method_match_test.go @@ -103,24 +103,25 @@ var _ = Describe("HTTPRoute method matches", func() { //get the pods of deployment1 pods := testFramework.GetPodsByDeploymentName(deployment1.Name, deployment1.Namespace) + pod := pods[0] Eventually(func(g Gomega) { cmd := fmt.Sprintf("curl -X GET %s", dnsName) - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd, true) + stdout, _, err := testFramework.PodExec(pod, cmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("test-get handler pod")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) Eventually(func(g Gomega) { cmd := fmt.Sprintf("curl -X POST %s", dnsName) - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd, true) + stdout, _, err := testFramework.PodExec(pod, cmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("test-post handler pod")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) Eventually(func(g Gomega) { invalidCmd := fmt.Sprintf("curl -X DELETE %s", dnsName) - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, invalidCmd, true) + stdout, _, err := testFramework.PodExec(pod, invalidCmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("Not Found")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) diff --git a/test/suites/integration/httproute_path_match_test.go b/test/suites/integration/httproute_path_match_test.go index b6e57eb9..d24ba3dd 100644 --- a/test/suites/integration/httproute_path_match_test.go +++ b/test/suites/integration/httproute_path_match_test.go @@ -131,18 +131,18 @@ var _ = Describe("HTTPRoute path matches", func() { //get the pods of deployment1 pods := testFramework.GetPodsByDeploymentName(deployment1.Name, deployment1.Namespace) Expect(len(pods)).To(BeEquivalentTo(1)) - log.Println("pods[0].Name:", pods[0].Name) + pod := pods[0] Eventually(func(g Gomega) { cmd := fmt.Sprintf("curl %s/pathmatch0", dnsName) - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd, true) + stdout, _, err := testFramework.PodExec(pod, cmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("test-v1 handler pod")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) Eventually(func(g Gomega) { cmd := fmt.Sprintf("curl %s/pathmatch1", dnsName) - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd, true) + stdout, _, err := testFramework.PodExec(pod, cmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("test-v2 handler pod")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) diff --git a/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go b/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go index a80fe598..2e71a1b4 100644 --- a/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go +++ b/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go @@ -122,7 +122,8 @@ var _ = Describe("Test 2 listeners with weighted httproute rules and service exp pods := testFramework.GetPodsByDeploymentName(deployment0.Name, deployment0.Namespace) Expect(len(pods)).To(BeEquivalentTo(1)) - log.Println("client pod name:", pods[0].Name) + pod := pods[0] + protocols := []string{"http", "https"} for _, protocol := range protocols { // just make sure we can reach via both protocols @@ -136,7 +137,7 @@ var _ = Describe("Test 2 listeners with weighted httproute rules and service exp } Eventually(func(g Gomega) { - stdout, _, err := testFramework.PodExec(pods[0].Namespace, pods[0].Name, cmd, true) + stdout, _, err := testFramework.PodExec(pod, cmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("handler pod")) }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index 6619f14a..55d23cc2 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -2,7 +2,6 @@ package integration import ( "context" - "flag" "os" "github.com/aws/aws-sdk-go/service/vpclattice" @@ -48,7 +47,7 @@ var _ = BeforeSuite(func() { testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway) - test.Logger(ctx).Infof("Expecting VPC %s and service network %s association", vpcId, *testServiceNetwork.Id) + testFramework.Log.Infof("Expecting VPC %s and service network %s association", vpcId, *testServiceNetwork.Id) Eventually(func(g Gomega) { associated, _, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcId, testServiceNetwork) g.Expect(associated).To(BeTrue()) @@ -56,11 +55,9 @@ var _ = BeforeSuite(func() { }) func TestIntegration(t *testing.T) { - var debug bool ctx = test.NewContext(t) - flag.BoolVar(&debug, "debug", false, "enable debug mode") - logger := gwlog.NewLogger(debug) - testFramework = test.NewFramework(ctx, logger.Named("framework"), k8snamespace) + logger := gwlog.NewLogger(true) + testFramework = test.NewFramework(ctx, logger, k8snamespace) RegisterFailHandler(Fail) RunSpecs(t, "Integration") }