From 74df6bae2533456e1c0d6fdf1beed5023d7168b5 Mon Sep 17 00:00:00 2001 From: Sascha Schwarze <schwarzs@de.ibm.com> Date: Fri, 22 Sep 2023 23:03:41 +0200 Subject: [PATCH] Setup webhook in integration test --- .github/workflows/ci.yml | 5 + Makefile | 12 +- hack/setup-webhook-cert-integration-test.sh | 81 +++++++++++++ test/integration/integration_suite_test.go | 16 ++- test/utils/webhook.go | 128 ++++++++++++++++++++ 5 files changed, 237 insertions(+), 5 deletions(-) create mode 100755 hack/setup-webhook-cert-integration-test.sh create mode 100644 test/utils/webhook.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b609386ef4..43a4894bd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,12 @@ jobs: kubectl -n tekton-pipelines rollout status deployment tekton-pipelines-webhook --timeout=1m - name: Test run: | + # host.docker.internal does not work in a GitHub action + docker exec kind-control-plane bash -c "echo '172.17.0.1 host.docker.internal' >>/etc/hosts" + + # Build and load the Git image export GIT_CONTAINER_IMAGE="$(KO_DOCKER_REPO=kind.local ko publish ./cmd/git)" + make test-integration e2e: diff --git a/Makefile b/Makefile index 057e50032c..127f6d23e4 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ TEST_NAMESPACE ?= default TEKTON_VERSION ?= v0.44.0 # E2E test flags -TEST_E2E_FLAGS ?= --fail-fast -p --randomize-all -timeout=1h -trace -vv +TEST_E2E_FLAGS ?= -p --randomize-all -timeout=1h -trace -v # E2E test service account name to be used for the build runs, can be set to generated to use the generated service account feature TEST_E2E_SERVICEACCOUNT_NAME ?= pipeline @@ -204,6 +204,7 @@ test-unit-ginkgo: ginkgo # Based on https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/integration-tests.md .PHONY: test-integration test-integration: install-apis ginkgo + ./hack/setup-webhook-cert-integration-test.sh $(GINKGO) \ --randomize-all \ --randomize-suites \ @@ -211,7 +212,6 @@ test-integration: install-apis ginkgo -trace \ test/integration/... - .PHONY: test-e2e test-e2e: install-strategies test-e2e-plain @@ -237,7 +237,13 @@ install-with-pprof: GOOS=$(GO_OS) GOARCH=$(GO_ARCH) GOFLAGS="$(GO_FLAGS) -tags=pprof_enabled" ko apply -R -f deploy/ -- --server-side install-apis: - kubectl apply -f deploy/crds/ --server-side + for resource in buildruns builds buildstrategies clusterbuildstrategies ; do \ + if kubectl get crd "$${resource}.shipwright.io" >/dev/null 2>&1 ; then \ + kubectl replace -f "deploy/crds/shipwright.io_$${resource}.yaml" ; \ + else \ + kubectl create -f "deploy/crds/shipwright.io_$${resource}.yaml" ; \ + fi ; \ + done for i in 1 2 3 ; do \ kubectl wait --timeout=$(TIMEOUT) --for="condition=Established" crd/clusterbuildstrategies.shipwright.io && \ break ; \ diff --git a/hack/setup-webhook-cert-integration-test.sh b/hack/setup-webhook-cert-integration-test.sh new file mode 100755 index 0000000000..fa795d5b06 --- /dev/null +++ b/hack/setup-webhook-cert-integration-test.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Copyright The Shipwright Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if ! hash jq >/dev/null 2>&1 ; then + echo "[ERROR] jq is not installed" + exit 1 +fi + +if ! hash openssl >/dev/null 2>&1 ; then + echo "[ERROR] openssl is not installed" + exit 1 +fi + +echo "[INFO] Generating key and signing request for Shipwright Build Webhook" + +cat <<EOF >/tmp/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = host.docker.internal +EOF + +openssl genrsa -out /tmp/server-key.pem 2048 +openssl req -new -days 365 -key /tmp/server-key.pem -subj "/O=system:nodes/CN=system:node:host.docker.internal" -out /tmp/server.csr -config /tmp/csr.conf + +echo "[INFO] Deleting previous CertificateSigningRequest" +kubectl delete csr shipwright-build-webhook-csr --ignore-not-found + +echo "[INFO] Create a CertificateSigningRequest" +cat <<EOF | kubectl create -f - +apiVersion: certificates.k8s.io/v1 +kind: CertificateSigningRequest +metadata: + name: shipwright-build-webhook-csr +spec: + groups: + - system:authenticated + request: $(base64 </tmp/server.csr | tr -d '\n') + signerName: kubernetes.io/kubelet-serving + usages: + - digital signature + - key encipherment + - server auth +EOF + +echo "[INFO] Approve the CertificateSigningRequest" +kubectl certificate approve shipwright-build-webhook-csr + +certificate="$(kubectl get csr shipwright-build-webhook-csr -o json | jq -r '.status.certificate')" +while [ "${certificate}" == "null" ]; do + echo "[INFO] Waiting for certificate to be ready" + sleep 1 + certificate="$(kubectl get csr shipwright-build-webhook-csr -o json | jq -r '.status.certificate')" +done + +openssl base64 -d -A -out /tmp/server-cert.pem <<<"${certificate}" + +echo "[INFO] Deleting the CertificateSigningRequest" +kubectl delete csr shipwright-build-webhook-csr --ignore-not-found +rm -rf /tmp/csr.conf + +echo "[INFO] Retrieving CABundle" +CA="$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n')" + +echo "[INFO] Patching caBundle into CustomResourceDefinitions" +kubectl patch crd clusterbuildstrategies.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]" +kubectl patch crd buildstrategies.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]" +kubectl patch crd builds.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]" +kubectl patch crd buildruns.shipwright.io --type=json -p "[{\"op\":\"replace\",\"path\":\"/spec/conversion/webhook/clientConfig\",\"value\":{\"caBundle\":\"${CA}\",\"url\":\"https://host.docker.internal:30443/convert\"}}]" diff --git a/test/integration/integration_suite_test.go b/test/integration/integration_suite_test.go index 7fa9f095a1..5a21bc314f 100644 --- a/test/integration/integration_suite_test.go +++ b/test/integration/integration_suite_test.go @@ -6,6 +6,7 @@ package integration_test import ( "fmt" + "net/http" "testing" . "github.com/onsi/ginkgo/v2" @@ -28,10 +29,21 @@ func TestIntegration(t *testing.T) { // TODO: clean resources in cluster, e.g. mainly cluster-scope ones // TODO: clean each resource created per spec var ( - tb *utils.TestBuild - err error + tb *utils.TestBuild + err error + webhookServer *http.Server ) +var _ = BeforeSuite(func() { + webhookServer = utils.StartBuildWebhook() +}) + +var _ = AfterSuite(func() { + if webhookServer != nil { + utils.StopBuildWebhook(webhookServer) + } +}) + var _ = BeforeEach(func() { tb, err = utils.NewTestBuild() if err != nil { diff --git a/test/utils/webhook.go b/test/utils/webhook.go new file mode 100644 index 0000000000..55402a4873 --- /dev/null +++ b/test/utils/webhook.go @@ -0,0 +1,128 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "context" + "crypto/tls" + "net/http" + "time" + + "github.com/shipwright-io/build/pkg/webhook/conversion" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +func StartBuildWebhook() *http.Server { + mux := http.NewServeMux() + mux.HandleFunc("/convert", conversion.CRDConvertHandler(context.Background())) + mux.HandleFunc("/health", health) + + webhookServer := &http.Server{ + Addr: ":30443", + Handler: mux, + ReadHeaderTimeout: 32 * time.Second, + IdleTimeout: time.Second, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.X25519}, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + }, + }, + } + + // start server + go func() { + defer ginkgo.GinkgoRecover() + + if err := webhookServer.ListenAndServeTLS("/tmp/server-cert.pem", "/tmp/server-key.pem"); err != nil { + if err != http.ErrServerClosed { + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + } + }() + + client := &http.Client{ + Transport: &http.Transport{ + IdleConnTimeout: 5 * time.Second, + ResponseHeaderTimeout: 5 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.X25519}, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + }, + }, + TLSHandshakeTimeout: 5 * time.Second, + }, + } + + gomega.Eventually(func() int { + r, err := client.Get("https://localhost:30443/health") + if err != nil { + return 0 + } + if r != nil { + return r.StatusCode + } + return 0 + }).WithTimeout(10 * time.Second).Should(gomega.Equal(http.StatusNoContent)) + + return webhookServer +} + +func StopBuildWebhook(webhookServer *http.Server) { + err := webhookServer.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + client := &http.Client{ + Transport: &http.Transport{ + IdleConnTimeout: 5 * time.Second, + ResponseHeaderTimeout: 5 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + CurvePreferences: []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.X25519}, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + }, + }, + TLSHandshakeTimeout: 5 * time.Second, + }, + } + + gomega.Eventually(func() int { + r, err := client.Get("https://localhost:30443/health") + if err != nil { + return 0 + } + if r != nil { + return r.StatusCode + } + return 0 + }).WithTimeout(10 * time.Second).Should(gomega.Equal(0)) +} + +func health(resp http.ResponseWriter, _ *http.Request) { + resp.WriteHeader(http.StatusNoContent) +}