From d747a790786bf6fff45023df74f037143ce0304d Mon Sep 17 00:00:00 2001 From: Spolti Date: Fri, 8 Sep 2023 10:38:27 -0300 Subject: [PATCH] wip -- e2e for non webhook installation Signed-off-by: Spolti --- Makefile | 13 ++ config/default-no-webhooks/kustomization.yaml | 2 +- .../controller_manager_config.yaml | 11 + config/manager-no-webhooks/kustomization.yaml | 40 ++++ config/manager-no-webhooks/manager.yaml | 65 ++++++ .../sonataflow_builder_dockerfile.yaml | 31 +++ operator-no-webhooks.yaml | 2 + test/e2e/workflow_nowebhooks_test.go | 208 ++++++++++++++++++ test/e2e/workflow_test.go | 29 ++- 9 files changed, 398 insertions(+), 3 deletions(-) create mode 100644 config/manager-no-webhooks/controller_manager_config.yaml create mode 100644 config/manager-no-webhooks/kustomization.yaml create mode 100644 config/manager-no-webhooks/manager.yaml create mode 100644 config/manager-no-webhooks/sonataflow_builder_dockerfile.yaml create mode 100644 test/e2e/workflow_nowebhooks_test.go diff --git a/Makefile b/Makefile index 3d5b7d364..943620499 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,10 @@ help: ## Display this help. manifests: generate ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) rbac:roleName=manager-role crd:allowDangerousTypes=true webhook paths="./api/..." paths="./controllers/..." output:crd:artifacts:config=config/crd/bases +.PHONY: manifests-crd-no-webhooks +manifests-crd-no-webhooks: generate ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd:allowDangerousTypes=true webhook paths="./api/..." paths="./controllers/..." output:crd:artifacts:config=config/crd-no-webhooks/bases + .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." paths="./container-builder/api/..." @@ -203,6 +207,10 @@ endif install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl create -f - +.PHONY: install-no-webhooks +install-no-webhooks: manifests-crd-no-webhooks kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd-no-webhooks | kubectl apply -f - + .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - @@ -212,6 +220,11 @@ deploy: manifests kustomize install-cert-manager ## Deploy controller to the K8s cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl create -f - +.PHONY: deploy-no-webhooks +deploy-no-webhooks: manifests-crd-no-webhooks kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default-no-webhooks | kubectl apply -f - + .PHONY: undeploy undeploy: uninstall-cert-manager ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - diff --git a/config/default-no-webhooks/kustomization.yaml b/config/default-no-webhooks/kustomization.yaml index e00cbb239..94039d3a3 100644 --- a/config/default-no-webhooks/kustomization.yaml +++ b/config/default-no-webhooks/kustomization.yaml @@ -15,7 +15,7 @@ namePrefix: sonataflow-operator- bases: - ../crd-no-webhooks - ../rbac -- ../manager +- ../manager-no-webhooks # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus diff --git a/config/manager-no-webhooks/controller_manager_config.yaml b/config/manager-no-webhooks/controller_manager_config.yaml new file mode 100644 index 000000000..4cefd89bb --- /dev/null +++ b/config/manager-no-webhooks/controller_manager_config.yaml @@ -0,0 +1,11 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: 1be5e57d.kiegroup.org diff --git a/config/manager-no-webhooks/kustomization.yaml b/config/manager-no-webhooks/kustomization.yaml new file mode 100644 index 000000000..e1e555db2 --- /dev/null +++ b/config/manager-no-webhooks/kustomization.yaml @@ -0,0 +1,40 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +- files: + - Dockerfile=sonataflow_builder_dockerfile.yaml + literals: + - DEFAULT_BUILDER_RESOURCE_NAME=Dockerfile + - DEFAULT_WORKFLOW_EXTENSION=.sw.json + name: builder-config + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/kiegroup/kogito-serverless-operator-nightly + newTag: latest +patchesJson6902: +- patch: |- + - op: add + path: /spec/template/spec/containers/0/env + value: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_WEBHOOKS + value: "false" + target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 diff --git a/config/manager-no-webhooks/manager.yaml b/config/manager-no-webhooks/manager.yaml new file mode 100644 index 000000000..b01554af6 --- /dev/null +++ b/config/manager-no-webhooks/manager.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - command: + - /usr/local/bin/manager + args: + - --leader-elect + - --v=2 + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/manager-no-webhooks/sonataflow_builder_dockerfile.yaml b/config/manager-no-webhooks/sonataflow_builder_dockerfile.yaml new file mode 100644 index 000000000..90b4a40e8 --- /dev/null +++ b/config/manager-no-webhooks/sonataflow_builder_dockerfile.yaml @@ -0,0 +1,31 @@ +FROM quay.io/kiegroup/kogito-swf-builder-nightly:latest AS builder + +# variables that can be overridden by the builder +# To add a Quarkus extension to your application +ARG QUARKUS_EXTENSIONS +# Args to pass to the Quarkus CLI add extension command +ARG QUARKUS_ADD_EXTENSION_ARGS + +# Copy from build context to skeleton resources project +COPY --chmod=644 * ./resources/ + +RUN /home/kogito/launch/build-app.sh ./resources + +#============================= +# Runtime Run +#============================= +FROM registry.access.redhat.com/ubi8/openjdk-11:latest + +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --from=builder --chown=185 /home/kogito/serverless-workflow-project/target/quarkus-app/lib/ /deployments/lib/ +COPY --from=builder --chown=185 /home/kogito/serverless-workflow-project/target/quarkus-app/*.jar /deployments/ +COPY --from=builder --chown=185 /home/kogito/serverless-workflow-project/target/quarkus-app/app/ /deployments/app/ +COPY --from=builder --chown=185 /home/kogito/serverless-workflow-project/target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +USER 185 +ENV AB_JOLOKIA_OFF="" +ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/operator-no-webhooks.yaml b/operator-no-webhooks.yaml index d8dd93d53..bf2c986cd 100644 --- a/operator-no-webhooks.yaml +++ b/operator-no-webhooks.yaml @@ -3592,6 +3592,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: ENABLE_WEBHOOKS + value: "false" image: quay.io/kiegroup/kogito-serverless-operator-nightly:latest livenessProbe: httpGet: diff --git a/test/e2e/workflow_nowebhooks_test.go b/test/e2e/workflow_nowebhooks_test.go new file mode 100644 index 000000000..28ed4a00b --- /dev/null +++ b/test/e2e/workflow_nowebhooks_test.go @@ -0,0 +1,208 @@ +// Copyright 2022 Red Hat, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "os/exec" + "path/filepath" + "time" + + "github.com/kiegroup/kogito-serverless-operator/test" + "github.com/kiegroup/kogito-serverless-operator/test/utils" + + //nolint:golint + //nolint:revive + . "github.com/onsi/ginkgo/v2" + + //nolint:golint + //nolint:revive + . "github.com/onsi/gomega" +) + +var _ = Describe("SonataFlow Operator - no webhooks", Serial, func() { + BeforeAll(func() { + + var controllerPodName string + operatorImageName, err := utils.GetOperatorImageName() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("installing CRDs") + cmd := exec.Command("make", "install-no-webhooks") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy-no-webhooks", fmt.Sprintf("IMG=%s", operatorImageName)) + + outputMake, err := utils.Run(cmd) + fmt.Println(string(outputMake)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + /* // TODO: Uncomment to enable when https://issues.redhat.com/browse/KOGITO-9110 will be available + + By("validating that manager Pod/container(s) are restricted") + // Get Podsecurity violation lines + lines, err := utils.StringToLines(string(outputMake)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + var violationLines []string + applySeccompProfilePatch := false + for _, line := range lines { + if strings.Contains(line, "Warning: would violate PodSecurity") { + if strings.Contains(line, "must set securityContext.seccompProfile.type to") { + // Ignore this violation as it is expected + applySeccompProfilePatch = true + } else { + violationLines = append(violationLines, line) + } + } + } + Expect(violationLines).To(BeEmpty()) + + if applySeccompProfilePatch { + By("Applying seccompProfile") + cmd = exec.Command("kubectl", "patch", "deployment", "sonataflow-operator-controller-manager", "-p", `{"spec":{"template":{"spec":{"securityContext":{"seccompProfile":{"type":"RuntimeDefault"}}}}}}`, "-n", nwhNamespace) + _, err := utils.Run(cmd) + if utils.IsDebugEnabled() { + err = utils.OutputDeployment(nwhNamespace, "sonataflow-operator-controller-manager") + } + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + */ + + By("validating that the controller-manager pod is running as expected - no webhooks") + verifyControllerUp := func() error { + var podOutput []byte + var err error + + if utils.IsDebugEnabled() { + err = utils.OutputAllPods() + err = utils.OutputAllEvents(namespace) + } + + // Get pod name + cmd = exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + podOutput, err = utils.Run(cmd) + fmt.Println(string(podOutput)) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + podNames := utils.GetNonEmptyLines(string(podOutput)) + if len(podNames) != 1 { + return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) + } + controllerPodName = podNames[0] + ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + + // Validate pod status + cmd = exec.Command("kubectl", "get", + "pods", controllerPodName, "-o", "jsonpath={.status.phase}", + "-n", namespace, + ) + status, err := utils.Run(cmd) + fmt.Println(string(status)) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + if string(status) != "Running" { + return fmt.Errorf("controller pod in %s status", status) + } + return nil + } + EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + }) + + AfterAll(func() { + By("removing manager namespace - no webhooks") + cmd := exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + //deleteNamespaceAndWait() + }) + + Describe("ensure that Operator and Operand(s) can run in restricted namespaces - no webhooks", func() { + projectDir, _ := utils.GetProjectDir() + + It("should create a basic platform for Minikube - no webhooks", func() { + By("creating an instance of the SonataFlowPlatform") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + getSonataFlowPlatformFilename()), "-n", namespace) + _, err := utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + }) + + It("should successfully deploy the Greeting Workflow in prod mode and verify if it's running - no webhooks", func() { + By("creating external resources DataInputSchema configMap - no webhooks") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + "test/testdata/"+test.SonataFlowGreetingsDataInputSchemaConfig), "-n", namespace) + _, err := utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("creating an instance of the SonataFlow Operand(CR) - no webhooks") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + "test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", namespace) + _, err := utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("check the workflow is in running state - no webhooks") + EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("greeting") }, 15*time.Minute, 30*time.Second).Should(BeTrue()) + + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir, + "test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", namespace) + _, err := utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + }) + + It("should successfully deploy the orderprocessing workflow in devmode and verify if it's running - no webhooks", func() { + + By("creating an instance of the SonataFlow Workflow in DevMode - no webhooks") + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir, + test.GetSonataFlowE2eOrderProcessingFolder()), "-n", namespace) + _, err := utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("check the workflow is in running state - no webhooks") + EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("orderprocessing") }, 5*time.Minute, 30*time.Second).Should(BeTrue()) + + cmdLog := exec.Command("kubectl", "logs", "orderprocessing", "-n", namespace) + if responseLog, errLog := utils.Run(cmdLog); errLog == nil { + GinkgoWriter.Println(fmt.Sprintf("devmode podlog %s", responseLog)) + } + + By("check that the workflow is addressable - no webhooks") + EventuallyWithOffset(1, func() bool { return verifyWorkflowIsAddressable("orderprocessing") }, 5*time.Minute, 30*time.Second).Should(BeTrue()) + + EventuallyWithOffset(1, func() error { + cmd := exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir, + test.GetSonataFlowE2eOrderProcessingFolder()), "-n", namespace) + _, err := utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + }) + }) +}) diff --git a/test/e2e/workflow_test.go b/test/e2e/workflow_test.go index 7ccc3382d..0a0fefba3 100644 --- a/test/e2e/workflow_test.go +++ b/test/e2e/workflow_test.go @@ -44,10 +44,9 @@ const ( openshiftPlatform = "openshift" ) -var _ = Describe("SonataFlow Operator", Ordered, func() { +var _ = Describe("SonataFlow Operator", Serial, func() { BeforeAll(func() { - // Now, let's ensure that all namespaces can raise a Warn when we apply the manifests // and that the namespace where the Operator and Operand will run are enforced as // restricted so that we can ensure that both can be admitted and run with the enforcement @@ -166,6 +165,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { By("uninstalling CRDs") cmd = exec.Command("make", "uninstall") _, _ = utils.Run(cmd) + //deleteNamespaceAndWait() }) Describe("ensure that Operator and Operand(s) can run in restricted namespaces", func() { @@ -293,3 +293,28 @@ func getClusterPlatform() string { } return minikubePlatform } + +func deleteNamespaceAndWait() { + // make sure there is no namespace created + By("deleting manager namespace - no webhooks") + cmd := exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + + waitForNamespaceDeletion := func() bool { + // Replace this with your custom condition logic + cmd := exec.Command("kubectl", "get", "ns", namespace) + output, err := utils.Run(cmd) + + if strings.Contains(err.Error(), "NotFound") { + return true + } else { + fmt.Printf("###################### %s ", output) + } + + return false + } + + for waitForNamespaceDeletion() { + time.Sleep(5 * time.Second) + } +}