diff --git a/test/e2e-helm/e2e_helm_olm_test.go b/test/e2e-helm/e2e_helm_olm_test.go index 67260d6a857..a09631336f5 100644 --- a/test/e2e-helm/e2e_helm_olm_test.go +++ b/test/e2e-helm/e2e_helm_olm_test.go @@ -15,6 +15,8 @@ package e2e_helm_test import ( + "encoding/json" + "fmt" "os/exec" "path" "path/filepath" @@ -22,6 +24,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" testutils "github.com/operator-framework/operator-sdk/test/internal" ) @@ -30,6 +33,14 @@ var _ = Describe("Integrating Helm Projects with OLM", func() { Context("with operator-sdk", func() { const operatorVersion = "0.0.1" + const ( + OLMBundleValidationTest = "olm-bundle-validation" + OLMCRDsHaveValidationTest = "olm-crds-have-validation" + OLMCRDsHaveResourcesTest = "olm-crds-have-resources" + OLMSpecDescriptorsTest = "olm-spec-descriptors" + OLMStatusDescriptorsTest = "olm-status-descriptors" + ) + BeforeEach(func() { By("turning off interactive prompts for all generation tasks.") replace := "operator-sdk generate kustomize manifests" @@ -63,6 +74,19 @@ var _ = Describe("Integrating Helm Projects with OLM", func() { err = tc.Make("packagemanifests", "IMG="+tc.ImageName) Expect(err).NotTo(HaveOccurred()) + By("running basic scorecard tests") + var scorecardOutput v1alpha3.TestList + runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=basic", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err := tc.Run(runScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(len(scorecardOutput.Items)).To(Equal(1)) + Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + By("running the package") runPkgManCmd := exec.Command(tc.BinaryName, "run", "packagemanifests", "--install-mode", "AllNamespaces", @@ -76,6 +100,31 @@ var _ = Describe("Integrating Helm Projects with OLM", func() { "--timeout", "4m") _, err = tc.Run(cleanupPkgManCmd) Expect(err).NotTo(HaveOccurred()) + + By("running olm scorecard tests") + runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=olm", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) + Expect(err).To(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + + expected := make(map[string]v1alpha3.State) + expected[OLMBundleValidationTest] = v1alpha3.PassState + expected[OLMCRDsHaveResourcesTest] = v1alpha3.FailState + expected[OLMCRDsHaveValidationTest] = v1alpha3.FailState + expected[OLMSpecDescriptorsTest] = v1alpha3.FailState + expected[OLMStatusDescriptorsTest] = v1alpha3.FailState + + Expect(len(scorecardOutput.Items)).To(Equal(len(expected))) + for a := 0; a < len(scorecardOutput.Items); a++ { + fmt.Println(" - Name: ", scorecardOutput.Items[a].Status.Results[0].Name) + fmt.Println(" Expected: ", expected[scorecardOutput.Items[a].Status.Results[0].Name]) + fmt.Println(" Output: ", scorecardOutput.Items[a].Status.Results[0].State) + Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(expected[scorecardOutput.Items[a].Status.Results[0].Name])) + } }) }) })