From 561f8efe7a641ea9513cc5e61a86730dce5d9175 Mon Sep 17 00:00:00 2001 From: Laura Fitzgerald Date: Tue, 26 Sep 2023 16:45:28 +0100 Subject: [PATCH] verify wildcard cert on http get request --- test/e2e/gateway_single_spoke_test.go | 285 ++++++++++++++++++++------ test/util/helper.go | 18 +- test/util/suite_config.go | 2 +- test/util/test_types.go | 35 +++- 4 files changed, 275 insertions(+), 65 deletions(-) diff --git a/test/e2e/gateway_single_spoke_test.go b/test/e2e/gateway_single_spoke_test.go index 6a248dbbd..22e1eff8b 100644 --- a/test/e2e/gateway_single_spoke_test.go +++ b/test/e2e/gateway_single_spoke_test.go @@ -17,6 +17,7 @@ import ( ocm_cluster_v1beta1 "open-cluster-management.io/api/cluster/v1beta1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,6 +37,8 @@ var _ = Describe("Gateway single target cluster", func() { // tests can be run in parallel in the same cluster var testID string var hostname gatewayapi.Hostname + var otherHostname gatewayapi.Hostname + var wildcardHostname gatewayapi.Hostname var gw *gatewayapi.Gateway var placement *ocm_cluster_v1beta1.Placement var tlsPolicy *v1alpha1.TLSPolicy @@ -43,7 +46,7 @@ var _ = Describe("Gateway single target cluster", func() { BeforeEach(func(ctx SpecContext) { testID = "t-e2e-" + tconfig.GenerateName() - By("creating a Placement for the Gateway resource ") + By("creating a Placement for the Gateway resource") placement = &ocm_cluster_v1beta1.Placement{ ObjectMeta: metav1.ObjectMeta{Name: testID, Namespace: tconfig.HubNamespace()}, Spec: ocm_cluster_v1beta1.PlacementSpec{ @@ -57,33 +60,22 @@ var _ = Describe("Gateway single target cluster", func() { By("creating a Gateway in the hub") hostname = gatewayapi.Hostname(strings.Join([]string{testID, tconfig.ManagedZone()}, ".")) - gw = &gatewayapi.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: testID, - Namespace: tconfig.HubNamespace(), - Labels: map[string]string{"gw": "t-e2e"}, - }, - Spec: gatewayapi.GatewaySpec{ - GatewayClassName: GatewayClassName, - Listeners: []gatewayapi.Listener{{ - Name: "https", - Hostname: &hostname, - Port: 443, - Protocol: gatewayapi.HTTPSProtocolType, - TLS: &gatewayapi.GatewayTLSConfig{ - CertificateRefs: []gatewayapi.SecretObjectReference{{ - Name: gatewayapi.ObjectName(hostname), - }}, - }, - AllowedRoutes: &gatewayapi.AllowedRoutes{ - Namespaces: &gatewayapi.RouteNamespaces{ - From: Pointer(gatewayapi.NamespacesFromAll), - }, - }, + gw = NewTestGateway(testID, GatewayClassName, tconfig.HubNamespace()).WithListener(gatewayapi.Listener{ + Name: "https", + Hostname: &hostname, + Port: 443, + Protocol: gatewayapi.HTTPSProtocolType, + TLS: &gatewayapi.GatewayTLSConfig{ + CertificateRefs: []gatewayapi.SecretObjectReference{{ + Name: gatewayapi.ObjectName(hostname), }}, }, - } - + AllowedRoutes: &gatewayapi.AllowedRoutes{ + Namespaces: &gatewayapi.RouteNamespaces{ + From: Pointer(gatewayapi.NamespacesFromAll), + }, + }, + }).WithLabels(map[string]string{"gw": "t-e2e"}).Gateway err = tconfig.HubClient().Create(ctx, gw) Expect(err).ToNot(HaveOccurred()) @@ -130,11 +122,11 @@ var _ = Describe("Gateway single target cluster", func() { AfterEach(func(ctx SpecContext) { err := tconfig.HubClient().Delete(ctx, gw, client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(err).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) err = tconfig.HubClient().Delete(ctx, tlsPolicy, client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(err).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) //Workaround for https://github.com/Kuadrant/multicluster-gateway-controller/issues/420 Eventually(func(ctx SpecContext) error { @@ -143,14 +135,14 @@ var _ = Describe("Gateway single target cluster", func() { err = tconfig.HubClient().Delete(ctx, placement, client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(err).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) err = tconfig.SpokeClient(0).Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testID}}, client.PropagationPolicy(metav1.DeletePropagationForeground)) - Expect(err).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) }) - When("the controller picks it up", func() { + When("the controller picks the gateway up", func() { It("sets the 'Accepted' conditions to true and programmed condition to unknown", func(ctx SpecContext) { @@ -163,7 +155,7 @@ var _ = Describe("Gateway single target cluster", func() { Expect(err).ToNot(HaveOccurred()) if !meta.IsStatusConditionPresentAndEqual(gw.Status.Conditions, string(gatewayapi.GatewayConditionProgrammed), "Unknown") { cond := meta.FindStatusCondition(gw.Status.Conditions, string(gatewayapi.GatewayConditionProgrammed)) - return fmt.Errorf("Expected condition %s to be Unknown but got %v", string(gatewayapi.GatewayConditionProgrammed), cond) + return fmt.Errorf("expected condition %s to be Unknown but got %v", string(gatewayapi.GatewayConditionProgrammed), cond) } return nil }).WithContext(ctx).WithTimeout(10 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) @@ -172,7 +164,7 @@ var _ = Describe("Gateway single target cluster", func() { err := tconfig.HubClient().Get(ctx, client.ObjectKey{Name: testID, Namespace: tconfig.HubNamespace()}, gw) Expect(err).ToNot(HaveOccurred()) if !meta.IsStatusConditionTrue(gw.Status.Conditions, string(gatewayapi.GatewayConditionAccepted)) { - return fmt.Errorf("Expected condition %s to be true", string(gatewayapi.GatewayConditionAccepted)) + return fmt.Errorf("expected condition %s to be true", string(gatewayapi.GatewayConditionAccepted)) } return nil }).WithContext(ctx).WithTimeout(10 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) @@ -190,7 +182,7 @@ var _ = Describe("Gateway single target cluster", func() { Expect(err).ToNot(HaveOccurred()) }) - It("it is placed on the spoke cluster once the tls secrets exist", func(ctx SpecContext) { + It("the gateway is placed on the spoke cluster once the tls secrets exist", func(ctx SpecContext) { istioGW := &gatewayapi.Gateway{} Eventually(func(ctx SpecContext) error { return tconfig.SpokeClient(0).Get(ctx, client.ObjectKey{Name: testID, Namespace: tconfig.SpokeNamespace()}, istioGW) @@ -238,7 +230,7 @@ var _ = Describe("Gateway single target cluster", func() { AfterEach(func(ctx SpecContext) { err := tconfig.SpokeClient(0).Delete(ctx, httproute) - Expect(err).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) }) When("a DNSPolicy is attached to the Gateway", func() { @@ -268,36 +260,45 @@ var _ = Describe("Gateway single target cluster", func() { AfterEach(func(ctx SpecContext) { err := tconfig.HubClient().Delete(ctx, dnsPolicy) - Expect(err).ToNot(HaveOccurred()) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: strings.Join([]string{testID, tconfig.ManagedZone()}, "."), + Namespace: tconfig.HubNamespace(), + }, + } + err = tconfig.HubClient().Delete(ctx, secret) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + }) It("makes available a hostname that can be resolved and reachable through HTTPS", func(ctx SpecContext) { - // Wait for the DNSrecord to exists: this shouldn't be necessary but I have found out that AWS dns servers - // cache the "no such host" response for a period of aprox 10-15 minutes, which makes the test run for a + // Wait for the DNSRecord to exists: this shouldn't be necessary, but I have found out that AWS dns servers + // cache the "no such host" response for a period of approx 10-15 minutes, which makes the test run for a // long time. By("waiting for the DNSRecord to be created in the Hub") - - Eventually(func(ctx SpecContext) bool { - dnsrecord := &mgcv1alpha1.DNSRecord{ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", gw.Name, "https"), - Namespace: gw.Namespace, - }, - } - if err := tconfig.HubClient().Get(ctx, client.ObjectKeyFromObject(dnsrecord), dnsrecord); err != nil { - GinkgoWriter.Printf("[debug] unable to get DNSRecord: '%s'\n", err) - return false + { + Eventually(func(ctx SpecContext) bool { + dnsrecord := &mgcv1alpha1.DNSRecord{ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", gw.Name, "https"), + Namespace: gw.Namespace, + }, + } + if err := tconfig.HubClient().Get(ctx, client.ObjectKeyFromObject(dnsrecord), dnsrecord); err != nil { + GinkgoWriter.Printf("[debug] unable to get DNSRecord: '%s'\n", err) + return false + } + return meta.IsStatusConditionTrue(dnsrecord.Status.Conditions, string(conditions.ConditionTypeReady)) + }).WithTimeout(300 * time.Second).WithPolling(10 * time.Second).WithContext(ctx).Should(BeTrue()) + + // still need to wait some seconds to the dns server to actually start + // resolving the hostname + select { + case <-ctx.Done(): + case <-time.After(30 * time.Second): } - return meta.IsStatusConditionTrue(dnsrecord.Status.Conditions, string(conditions.ConditionTypeReady)) - }).WithTimeout(300 * time.Second).WithPolling(10 * time.Second).WithContext(ctx).Should(BeTrue()) - - // still need to wait some seconds to the dns server to actually start - // resolving the hostname - select { - case <-ctx.Done(): - case <-time.After(30 * time.Second): } - By("ensuring the authoritative nameserver resolves the hostname") // speed up things by using the authoritative nameserver @@ -332,26 +333,186 @@ var _ = Describe("Gateway single target cluster", func() { } http.DefaultTransport.(*http.Transport).DialContext = dialContext http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - httpClient := &http.Client{} var resp *http.Response Eventually(func(ctx SpecContext) error { + httpClient := &http.Client{} resp, err = httpClient.Get("https://" + string(hostname)) if err != nil { GinkgoWriter.Printf("[debug] GET error: '%s'\n", err) return err } - return nil + return TestCertificate(string(hostname), resp) }).WithTimeout(300 * time.Second).WithPolling(10 * time.Second).WithContext(ctx).ShouldNot(HaveOccurred()) defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) } - }) + By("adding wildcard listener to the gateway") + { + gw := &gatewayapi.Gateway{} + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: testID, Namespace: tconfig.HubNamespace()}, gw) + Expect(err).ToNot(HaveOccurred()) + + if gw.Spec.Listeners == nil { + gw.Spec.Listeners = []gatewayapi.Listener{} + } + wildcardHostname = gatewayapi.Hostname(strings.Join([]string{"*", tconfig.ManagedZone()}, ".")) + secretName := gatewayapi.Hostname(strings.Join([]string{testID, tconfig.ManagedZone()}, ".")) + AddListener("wildcard", wildcardHostname, gatewayapi.ObjectName(secretName), gw) + err = tconfig.HubClient().Update(ctx, gw) + Expect(err).ToNot(HaveOccurred()) + expectedListeners := 2 + Eventually(func(ctx SpecContext) error { + checkGateway := &gatewayapi.Gateway{} + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: testID, Namespace: tconfig.HubNamespace()}, checkGateway) + Expect(err).ToNot(HaveOccurred()) + if len(checkGateway.Spec.Listeners) == expectedListeners { + return nil + } + return fmt.Errorf("should be %d listeners in the gateway", expectedListeners) + }).WithContext(ctx).WithTimeout(100 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) + + } + + By("checking tls secrets") + { + secretList := &corev1.SecretList{} + Eventually(func(ctx SpecContext) error { + err = tconfig.HubClient().List(ctx, secretList) + if err != nil { + return err + } + if len(secretList.Items) == 0 { + return fmt.Errorf("no secrets found") + } + for _, secret := range secretList.Items { + if secret.Name == string(hostname) { + if strings.Contains(secret.Annotations["cert-manager.io/alt-names"], string(hostname)) && + strings.Contains(secret.Annotations["cert-manager.io/alt-names"], string(wildcardHostname)) { + return nil + } + } + } + return fmt.Errorf("dns names for secret not as expected") + }).WithContext(ctx).WithTimeout(180 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) + } + By("checking a wildcard cert is present via get request") + { + dialer := &net.Dialer{Resolver: authoritativeResolver} + dialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { + return dialer.DialContext(ctx, network, addr) + } + http.DefaultTransport.(*http.Transport).DialContext = dialContext + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + otherHostname = gatewayapi.Hostname(strings.Join([]string{"other", tconfig.ManagedZone()}, ".")) + var resp *http.Response + Eventually(func(ctx SpecContext) error { + httpClient := &http.Client{} + resp, err = httpClient.Get("https://" + string(otherHostname)) + if err != nil { + GinkgoWriter.Printf("[debug] GET error: '%s'\n", err) + return err + } + err = TestCertificate(string(wildcardHostname), resp) + if err != nil { + GinkgoWriter.Printf("[debug] Cert error: '%s'\n", err) + return err + } + return nil + }).WithTimeout(600 * time.Second).WithPolling(10 * time.Second).WithContext(ctx).ShouldNot(HaveOccurred()) + defer resp.Body.Close() + } + By("adding/removing listeners tls secrets are added/removed") + { + gw := &gatewayapi.Gateway{} + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: testID, Namespace: tconfig.HubNamespace()}, gw) + Expect(err).ToNot(HaveOccurred()) + + if gw.Spec.Listeners == nil { + gw.Spec.Listeners = []gatewayapi.Listener{} + } + otherHostname = gatewayapi.Hostname(strings.Join([]string{"other", tconfig.ManagedZone()}, ".")) + AddListener("other", otherHostname, gatewayapi.ObjectName(otherHostname), gw) + err = tconfig.HubClient().Update(ctx, gw) + Expect(err).ToNot(HaveOccurred()) + expectedLiseners := 3 + Eventually(func(ctx SpecContext) error { + checkGateway := &gatewayapi.Gateway{} + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: testID, Namespace: tconfig.HubNamespace()}, checkGateway) + Expect(err).ToNot(HaveOccurred()) + if len(checkGateway.Spec.Listeners) == expectedLiseners { + return nil + } + return fmt.Errorf("expected %d listeners in the ", expectedLiseners) + }).WithContext(ctx).WithTimeout(100 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) + + Eventually(func(ctx SpecContext) error { + secret := &corev1.Secret{} + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: string(otherHostname), Namespace: tconfig.HubNamespace()}, secret) + if err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("secret %s not found", string(otherHostname)) + } + return err + } + return nil + }).WithContext(ctx).WithTimeout(180 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) + + // remove the listener + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: testID, Namespace: tconfig.HubNamespace()}, gw) + Expect(err).ToNot(HaveOccurred()) + + if gw.Spec.Listeners == nil { + gw.Spec.Listeners = []gatewayapi.Listener{} + } + + for i, listener := range gw.Spec.Listeners { + if listener.Name == "other" { + gw.Spec.Listeners = append(gw.Spec.Listeners[:i], gw.Spec.Listeners[i+1:]...) + } + } + err = tconfig.HubClient().Update(ctx, gw) + Expect(err).ToNot(HaveOccurred()) + Eventually(func(ctx SpecContext) error { + secret := &corev1.Secret{} + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: string(otherHostname), Namespace: tconfig.HubNamespace()}, secret) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("secret %s found, should be removed", otherHostname) + }).WithContext(ctx).WithTimeout(180 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) + } + By("deleting tls policy, tls secrets are removed") + { + tlsPolicy = &mgcv1alpha1.TLSPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: testID, + Namespace: tconfig.HubNamespace(), + }, + } + err := tconfig.HubClient().Delete(ctx, tlsPolicy, + client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(err).ToNot(HaveOccurred()) + hostname = gatewayapi.Hostname(strings.Join([]string{testID, tconfig.ManagedZone()}, ".")) + Eventually(func(ctx SpecContext) error { + secret := &corev1.Secret{} + err = tconfig.HubClient().Get(ctx, client.ObjectKey{Name: string(hostname), Namespace: tconfig.HubNamespace()}, secret) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("secret %s found, should be removed", hostname) + }).WithContext(ctx).WithTimeout(180 * time.Second).WithPolling(2 * time.Second).ShouldNot(HaveOccurred()) + } + }) }) }) - }) - }) diff --git a/test/util/helper.go b/test/util/helper.go index 2771d5cdd..0329d675c 100644 --- a/test/util/helper.go +++ b/test/util/helper.go @@ -1,8 +1,10 @@ -//go:build unit || integration +//go:build unit || integration || e2e package testutil import ( + "fmt" + "net/http" "strings" "testing" @@ -133,3 +135,17 @@ func GetBasicScheme() *runtime.Scheme { _ = corev1.AddToScheme(scheme) return scheme } + +func TestCertificate(dnsName string, resp *http.Response) error { + if resp.TLS == nil { + return fmt.Errorf("expected tls configuration to be present on http request") + } + for _, cert := range resp.TLS.PeerCertificates { + for _, name := range cert.DNSNames { + if name == dnsName { + return nil + } + } + } + return fmt.Errorf("wildcard hostname not found in the certificate via get request") +} diff --git a/test/util/suite_config.go b/test/util/suite_config.go index 5ad7b82b7..3b8d1ce9c 100644 --- a/test/util/suite_config.go +++ b/test/util/suite_config.go @@ -212,7 +212,7 @@ func (cfg *SuiteConfig) InstallPrerequisites(ctx context.Context) error { } } - // TODO ensure ManagedZone: right now this is added by local-setup + //TODO ensure ManagedZone: right now this is added by local-setup // ensure ManagedClusterSet { diff --git a/test/util/test_types.go b/test/util/test_types.go index 03537f9bf..667731098 100644 --- a/test/util/test_types.go +++ b/test/util/test_types.go @@ -1,4 +1,4 @@ -//go:build unit || integration +//go:build unit || integration || e2e package testutil @@ -52,11 +52,44 @@ func NewTestClusterIssuer(name string) *certmanv1.ClusterIssuer { } } +func AddListener(name string, hostname gatewayapiv1alpha2.Hostname, secretName gatewayv1beta1.ObjectName, gw *gatewayv1beta1.Gateway) { + listener := gatewayapiv1alpha2.Listener{ + Name: gatewayv1beta1.SectionName(name), + Hostname: &hostname, + Port: 443, + Protocol: gatewayv1beta1.HTTPSProtocolType, + TLS: &gatewayv1beta1.GatewayTLSConfig{ + CertificateRefs: []gatewayv1beta1.SecretObjectReference{ + { + Name: secretName, + }, + }, + }, + AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ + Namespaces: &gatewayv1beta1.RouteNamespaces{ + From: Pointer(gatewayv1beta1.NamespacesFromAll), + }, + }, + } + gw.Spec.Listeners = append(gw.Spec.Listeners, listener) + +} + func (t *TestGateway) WithListener(listener gatewayv1beta1.Listener) *TestGateway { t.Spec.Listeners = append(t.Spec.Listeners, listener) return t } +func (t *TestGateway) WithLabels(labels map[string]string) *TestGateway { + if t.Labels == nil { + t.Labels = map[string]string{} + } + for key, value := range labels { + t.Labels[key] = value + } + return t +} + func (t *TestGateway) WithHTTPListener(hostname string) *TestGateway { typedHostname := gatewayv1beta1.Hostname(hostname) t.WithListener(gatewayv1beta1.Listener{