From 926dbb398439acfb4fb578cda768455cf4d121b1 Mon Sep 17 00:00:00 2001 From: Amim Knabben Date: Sun, 10 Dec 2023 12:20:47 -0300 Subject: [PATCH] Adding reporter command line --- .../workflows/{unit-tests.yml => tests.yml} | 7 +- Makefile | 5 + cmd/reporter.go | 81 ++++++++++++++++ cmd/root.go | 60 +++++++----- go.mod | 25 +++-- go.sum | 49 ++++++---- hack/build_k8s_test_binary.sh | 2 - pkg/report/clean.go | 89 +++++++++++++++++ pkg/report/parse.go | 51 ++++++++++ pkg/report/types.go | 15 ++- pkg/testcases/cases.go | 97 +------------------ pkg/testcases/context_test.go | 52 ---------- tests/samples/junit_1101_fail.xml | 8 ++ tests/samples/junit_1101_valid.xml | 5 + tests/testcases/context.go | 39 ++++++++ ...indows_operational_readiness_suite_test.go | 15 +++ tests/xml/xml.go | 53 ++++++++++ 17 files changed, 452 insertions(+), 201 deletions(-) rename .github/workflows/{unit-tests.yml => tests.yml} (67%) create mode 100644 cmd/reporter.go create mode 100644 pkg/report/clean.go create mode 100644 pkg/report/parse.go delete mode 100644 pkg/testcases/context_test.go create mode 100644 tests/samples/junit_1101_fail.xml create mode 100644 tests/samples/junit_1101_valid.xml create mode 100644 tests/testcases/context.go create mode 100644 tests/windows_operational_readiness_suite_test.go create mode 100644 tests/xml/xml.go diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/tests.yml similarity index 67% rename from .github/workflows/unit-tests.yml rename to .github/workflows/tests.yml index 0bcfd51..b113c75 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/tests.yml @@ -18,9 +18,14 @@ jobs: with: go-version: 1.21 - uses: actions/checkout@v3 - - name: Run unit-tests + - name: Run unit tests run: | go test -v ./... + - name: Run bdd tests + run: | + go install github.com/onsi/ginkgo/v2/ginkgo + go get github.com/onsi/gomega/... + make test - name: Build the binary run: | make build diff --git a/Makefile b/Makefile index 0ebe01a..e3dcf1f 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,11 @@ build: ## Build the binary using local golang ./hack/build_k8s_test_binary.sh ${KUBERNETES_HASH} go build -o ./op-readiness . +.PHONY: test +test: ## Run the Sonobuoy plugin + pushd tests; ginkgo run -v; popd + + ## -------------------------------------- ## Container ## -------------------------------------- diff --git a/cmd/reporter.go b/cmd/reporter.go new file mode 100644 index 0000000..844ef22 --- /dev/null +++ b/cmd/reporter.go @@ -0,0 +1,81 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "log" + "strconv" + "strings" + + "github.com/spf13/cobra" + "sigs.k8s.io/windows-operational-readiness/pkg/report" +) + +var ( + directory string + failed int + passed int +) + +var reporterCmd = &cobra.Command{ + Use: "reporter", + Short: "Parse XML Junit results and print a columnar output", + Long: `Parse XML Junit results and print a columnar output`, + Run: func(cmd *cobra.Command, args []string) { + suites, err := report.ParseXMLFiles(directory) + if err != nil { + log.Fatal(err) + } + for _, n := range suites { + presentTestSuite(n) + } + fmt.Printf("Passed tests: %d\n", passed) + fmt.Printf("Failed tests: %d\n", failed) + }, +} + +func init() { + reporterCmd.Flags().StringVar(&directory, "dir", "", "directory to search in for xml files") +} + +func presentTestSuite(r *report.TestSuites) { + for _, t := range r.Suites { + if len(t.TestCases) == 0 { + ftime, err := strconv.ParseFloat(t.Time, 32) + if err != nil { + continue + } + fmt.Printf("[FAILED] | %s - %s | (%2.2fs) | %s\n", t.Index, r.Category, ftime, r.Name) + failed++ + } + + for _, c := range t.TestCases { + ftime, err := strconv.ParseFloat(c.Time, 32) + if err != nil { + continue + } + fmt.Printf("[%s] | %s - %s | (%2.2fs) | %s | %s\n", strings.ToUpper(string(c.Status)), t.Index, r.Category, ftime, r.Name, c.Name) + if c.Status == report.StatusPassed { + passed++ + } + if c.Status == report.StatusFailed { + failed++ + } + } + } +} diff --git a/cmd/root.go b/cmd/root.go index cb47c2f..d40f185 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,15 +19,30 @@ package cmd import ( "fmt" "os" + "path" "github.com/spf13/cobra" "go.uber.org/zap" "go.uber.org/zap/zapcore" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/windows-operational-readiness/pkg/flags" + "sigs.k8s.io/windows-operational-readiness/pkg/report" "sigs.k8s.io/windows-operational-readiness/pkg/testcases" ) +func init() { + rootCmd.PersistentFlags().StringVar(&testDirectory, "test-directory", "", "Path to YAML root directory containing the tests.") + rootCmd.PersistentFlags().StringVar(&E2EBinary, "e2e-binary", "./e2e.test", "The E2E Ginkgo default binary used to run the tests.") + rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, etc.)") + rootCmd.PersistentFlags().StringVar(&kubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") + rootCmd.PersistentFlags().StringVar(&reportDir, "report-dir", getEnvOrDefault("ARTIFACTS", ""), "Report dump directory, uses artifact for CI integration when set.") + rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Do not run actual tests, used for sanity check.") + rootCmd.PersistentFlags().BoolVar(&verbose, "verbose", false, "Enable Ginkgo verbosity.") + rootCmd.PersistentFlags().Var(&categories, "category", "Append category of tests you want to run, default empty will run all tests.") + + rootCmd.AddCommand(reporterCmd) +} + // NewLoggerConfig return the configuration object for the logger. func NewLoggerConfig(options ...zap.Option) *zap.Logger { core := zapcore.NewCore(zapcore.NewConsoleEncoder(zapcore.EncoderConfig{ @@ -64,7 +79,6 @@ var ( os.Exit(1) } testCtx := testcases.NewTestContext(E2EBinary, kubeConfig, provider, specs, dryRun, verbose, reportDir, categories) - targetedSpecs := make([]testcases.Specification, 0) for _, s := range specs { if !testCtx.CategoryEnabled(s.Category) { @@ -74,18 +88,31 @@ var ( } } - for sIdx, s := range targetedSpecs { - zap.L().Info(fmt.Sprintf("[OpReadinessTests] %d / %d Specifications - Running %d Test(s) for Ops Readiness specification: %v", sIdx+1, len(targetedSpecs), len(s.TestCases), s.Category)) + for specIdx, s := range targetedSpecs { + zap.L().Info(fmt.Sprintf("[OpReadinessTests] %d / %d Specifications - Running %d Test(s) for Ops Readiness specification: %v", specIdx+1, len(targetedSpecs), len(s.TestCases), s.Category)) + if len(s.TestCases) == 0 { zap.L().Info(fmt.Sprintf("[%s] No Operational Readiness tests to run", string(s.Category))) - } else { - for tIdx, t := range s.TestCases { - logPrefix := fmt.Sprintf("[%s] %v / %v Tests - ", s.Category, tIdx+1, len(s.TestCases)) - zap.L().Info(fmt.Sprintf("%sRunning Operational Readiness Test: %v", logPrefix, t.Description)) - if err = t.RunTest(testCtx, tIdx+1); err != nil { - zap.L().Error(fmt.Sprintf("%sFailed Operational Readiness Test: %v, error is %v", logPrefix, t.Description, zap.Error(err))) - } else { - zap.L().Info(fmt.Sprintf("%sPassed Operational Readiness Test: %v", logPrefix, t.Description)) + continue + } + + for testIdx, t := range s.TestCases { + prefix := fmt.Sprintf("%d%d", specIdx+1, testIdx+1) + logPrefix := fmt.Sprintf("[%s] %v / %v Tests - ", s.Category, testIdx+1, len(s.TestCases)) + zap.L().Info(fmt.Sprintf("%sRunning Operational Readiness Test: %v", logPrefix, t.Description)) + if err = t.RunTest(testCtx, prefix); err != nil { + zap.L().Error(fmt.Sprintf("%sFailed Operational Readiness Test: %v, error is %v", logPrefix, t.Description, zap.Error(err))) + } else { + zap.L().Info(fmt.Sprintf("%sPassed Operational Readiness Test: %v", logPrefix, t.Description)) + } + + // Specs already ran, cleaning up the XML Junit report + if testCtx.ReportDir != "" && !testCtx.DryRun { + fileName := "junit_" + prefix + "01.xml" + junitReport := path.Join(testCtx.ReportDir, fileName) + zap.L().Info("Cleaning XML files", zap.String("file", fileName), zap.String("path", junitReport)) + if err := report.CleanupJUnitXML(junitReport, string(s.Category), t.Description, testIdx+1); err != nil { + zap.L().Error(err.Error()) } } } @@ -105,14 +132,3 @@ func getEnvOrDefault(key, defaultString string) string { } return defaultString } - -func init() { - rootCmd.PersistentFlags().StringVar(&testDirectory, "test-directory", "", "Path to YAML root directory containing the tests.") - rootCmd.PersistentFlags().StringVar(&E2EBinary, "e2e-binary", "./e2e.test", "The E2E Ginkgo default binary used to run the tests.") - rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, etc.)") - rootCmd.PersistentFlags().StringVar(&kubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") - rootCmd.PersistentFlags().StringVar(&reportDir, "report-dir", getEnvOrDefault("ARTIFACTS", ""), "Report dump directory, uses artifact for CI integration when set.") - rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Do not run actual tests, used for sanity check.") - rootCmd.PersistentFlags().BoolVar(&verbose, "verbose", false, "Enable Ginkgo verbosity.") - rootCmd.PersistentFlags().Var(&categories, "category", "Append category of tests you want to run, default empty will run all tests.") -} diff --git a/go.mod b/go.mod index 57cb73a..39ece48 100644 --- a/go.mod +++ b/go.mod @@ -4,21 +4,25 @@ go 1.21 require ( github.com/go-playground/validator/v10 v10.10.0 + github.com/onsi/ginkgo/v2 v2.13.2 + github.com/onsi/gomega v1.29.0 github.com/spf13/cobra v1.4.0 go.uber.org/zap v1.20.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 k8s.io/client-go v0.23.3 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-logr/logr v1.2.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.6 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/imdario/mergo v0.3.5 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -28,15 +32,16 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.5.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.4.0 // indirect - golang.org/x/term v0.4.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/tools v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apimachinery v0.23.3 // indirect diff --git a/go.sum b/go.sum index 3396e31..91c3d4d 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= @@ -99,6 +100,8 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -130,8 +133,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -146,8 +150,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -165,6 +169,8 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -219,9 +225,13 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= +github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -276,8 +286,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -312,6 +322,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -350,8 +362,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -423,12 +435,12 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -438,8 +450,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -495,10 +507,11 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -599,8 +612,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -621,8 +635,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/build_k8s_test_binary.sh b/hack/build_k8s_test_binary.sh index 7c26f96..33087e1 100755 --- a/hack/build_k8s_test_binary.sh +++ b/hack/build_k8s_test_binary.sh @@ -16,9 +16,7 @@ set -o errexit set -o pipefail -set -x -# todo(knabben) - fetch latest or pass as argument KUBERNETES_VERSION=${KUBERNETES_VERSION:-"v1.28.0"} KUBERNETES_REPO=${KUBERNETES_REPO:-"https://github.com/kubernetes/kubernetes.git"} diff --git a/pkg/report/clean.go b/pkg/report/clean.go new file mode 100644 index 0000000..aa9efa2 --- /dev/null +++ b/pkg/report/clean.go @@ -0,0 +1,89 @@ +package report + +import ( + "encoding/xml" + "fmt" + "io" + "os" + + "go.uber.org/zap" +) + +// CleanupJUnitXML removes unnecessary skipped tests from the report. +func CleanupJUnitXML(path, category, testName string, index int) error { + zap.L().Info("Getting XML content from file", zap.String("path", path), zap.String("category", category)) + + content, err := getXMLContent(path) + if err != nil { + return err + } + + var testSuites TestSuites + if err = xml.Unmarshal(content, &testSuites); err != nil { + return err + } + for i, suite := range testSuites.Suites { + var cleaned []TestCase + for _, test := range suite.TestCases { + if test.Status != StatusSkipped && + test.Name != "[SynchronizedBeforeSuite]" && + test.Name != "[SynchronizedAfterSuite]" && + test.Name != "[ReportBeforeSuite]" && + test.Name != "[ReportAfterSuite] Kubernetes e2e suite report" { + cleaned = append(cleaned, test) + } + } + + zap.L().Info("Saving cleaned tests.", zap.Int("number", len(cleaned)), zap.Int("index", index)) + testSuites.Suites[i].Index = fmt.Sprint(index) + testSuites.Suites[i].TestCases = cleaned + } + + // save the category metadata + testSuites.Category = category + testSuites.Name = testName + + // write back the cleaned up YAML to a writer + var cleanContent []byte + if cleanContent, err = xml.MarshalIndent(testSuites, " ", " "); err != nil { + return err + } + + return writeFileContent(path, cleanContent) +} + +// writeFileContent save the content to a file. +func writeFileContent(path string, content []byte) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + if _, err = file.Write(content); err != nil { + return err + } + return nil +} + +// getXMLContent returns the content in bytes of an existent file. +func getXMLContent(path string) ([]byte, error) { + var err error + // check if file exists + if _, err = os.Stat(path); os.IsNotExist(err) { + return []byte{}, err + } + + var file *os.File + if file, err = os.Open(path); err != nil { + return []byte{}, err + } + defer file.Close() + + var content []byte + if content, err = io.ReadAll(file); err != nil { + return []byte{}, err + } + + return content, nil +} diff --git a/pkg/report/parse.go b/pkg/report/parse.go new file mode 100644 index 0000000..c23f959 --- /dev/null +++ b/pkg/report/parse.go @@ -0,0 +1,51 @@ +package report + +import ( + "encoding/xml" + "io" + "os" + "path" + "strings" +) + +func ParseXMLFiles(directory string) (testSuites []*TestSuites, err error) { + dir, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + for _, file := range dir { + name := file.Name() + if strings.HasSuffix(name, "xml") { + testSuite, err := UnmarshalXMLFile(path.Join(directory, name)) + if err != nil { + return nil, err + } + + testSuites = append(testSuites, testSuite) + } + } + return +} + +func UnmarshalXMLFile(fileName string) (*TestSuites, error) { + var content []byte + testSuites := &TestSuites{} + + fd, err := os.Open(fileName) + if err != nil { + return testSuites, err + } + defer fd.Close() + + content, err = io.ReadAll(fd) + if err != nil { + return testSuites, err + } + + err = xml.Unmarshal(content, testSuites) + if err != nil { + return testSuites, err + } + + return testSuites, err +} diff --git a/pkg/report/types.go b/pkg/report/types.go index 1b0950e..8bd5069 100644 --- a/pkg/report/types.go +++ b/pkg/report/types.go @@ -13,7 +13,7 @@ const ( StatusError Status = "error" ) -type TestCounter struct { +type Counters struct { Tests string `xml:"tests,attr"` Disabled string `xml:"disabled,attr"` Errors string `xml:"errors,attr"` @@ -22,9 +22,12 @@ type TestCounter struct { } type TestSuites struct { - XMLName xml.Name `xml:"testsuites"` - Suites []Suite `xml:"testsuite"` - TestCounter + XMLName xml.Name `xml:"testsuites"` + Suites []Suite `xml:"testsuite"` + Category string `xml:"category,attr"` + Name string `xml:"name,attr"` + + Counters } // Suite represents a logical grouping (suite) of tests. @@ -32,8 +35,10 @@ type Suite struct { XMLName xml.Name `xml:"testsuite"` Name string `xml:"name,attr"` Package string `xml:"package,attr"` + Index string `xml:"index,attr"` TestCases []TestCase `xml:"testcase,omitempty"` - TestCounter + + Counters } // TestCase represents the results of a single test run. diff --git a/pkg/testcases/cases.go b/pkg/testcases/cases.go index 7b2f8fe..b7cad35 100644 --- a/pkg/testcases/cases.go +++ b/pkg/testcases/cases.go @@ -18,15 +18,10 @@ package testcases import ( "bufio" - "encoding/xml" "io" - "os" "os/exec" - "path" - "strconv" "go.uber.org/zap" - "sigs.k8s.io/windows-operational-readiness/pkg/report" ) type Specification struct { @@ -56,8 +51,8 @@ const ( ) // RunTest runs the binary set in the test context with the parameters from flags. -func (t *TestCase) RunTest(testCtx *TestContext, idx int) error { - cmd := buildCmd(t, testCtx, idx) +func (t *TestCase) RunTest(testCtx *TestContext, prefix string) error { + cmd := buildCmd(t, testCtx, prefix) stdout, err := cmd.StdoutPipe() if err != nil { @@ -76,27 +71,15 @@ func (t *TestCase) RunTest(testCtx *TestContext, idx int) error { redirectOutput(stdout) redirectOutput(stderr) - if err := cmd.Wait(); err != nil { - return err - } - - if testCtx.ReportDir != "" && !testCtx.DryRun { - fileName := "junit_" + strconv.Itoa(idx) + "01.xml" - junitReport := path.Join(testCtx.ReportDir, fileName) - zap.L().Info("Cleaning XML files", zap.String("file", fileName), zap.String("path", junitReport)) - if err := CleanupJUnitXML(junitReport); err != nil { - return err - } - } - return nil + return cmd.Wait() } -func buildCmd(t *TestCase, testCtx *TestContext, idx int) *exec.Cmd { +func buildCmd(t *TestCase, testCtx *TestContext, prefix string) *exec.Cmd { args := []string{ "--provider", testCtx.Provider, "--kubeconfig", testCtx.KubeConfig, "--report-dir", testCtx.ReportDir, - "--report-prefix", strconv.Itoa(idx), + "--report-prefix", prefix, "--node-os-distro", "windows", "--non-blocking-taints", "os,node-role.kubernetes.io/master,node-role.kubernetes.io/control-plane", "--ginkgo.flakeAttempts", "1", @@ -136,76 +119,6 @@ func buildCmd(t *TestCase, testCtx *TestContext, idx int) *exec.Cmd { return exec.Command(testCtx.E2EBinary, args...) } -// CleanupJUnitXML removes unnecessary skipped tests from the report. -func CleanupJUnitXML(path string) error { - zap.L().Info("Getting XML content from file", zap.String("path", path)) - content, err := getXMLContent(path) - if err != nil { - return err - } - - var testSuites report.TestSuites - if err = xml.Unmarshal(content, &testSuites); err != nil { - return err - } - for i, suite := range testSuites.Suites { - var cleanTests []report.TestCase - for _, test := range suite.TestCases { - if test.Status != report.StatusSkipped && - test.Name != "[SynchronizedBeforeSuite]" && - test.Name != "[SynchronizedAfterSuite]" && - test.Name != "[ReportAfterSuite] Kubernetes e2e suite report" { - cleanTests = append(cleanTests, test) - } - } - testSuites.Suites[i].TestCases = cleanTests - zap.L().Info("Saving cleaned tests.", zap.Int("number", len(cleanTests))) - } - // write back the cleaned up YAML to a writer - var cleanContent []byte - if cleanContent, err = xml.MarshalIndent(testSuites, " ", " "); err != nil { - return err - } - - return writeFileContent(path, cleanContent) -} - -// writeFileContent save the content to a file. -func writeFileContent(path string, content []byte) error { - file, err := os.Create(path) - if err != nil { - return err - } - defer file.Close() - - if _, err = file.Write(content); err != nil { - return err - } - return nil -} - -// getXMLContent returns the content in bytes of an existent file. -func getXMLContent(path string) ([]byte, error) { - var err error - // check if file exists - if _, err = os.Stat(path); os.IsNotExist(err) { - return []byte{}, err - } - - var file *os.File - if file, err = os.Open(path); err != nil { - return []byte{}, err - } - defer file.Close() - - var content []byte - if content, err = io.ReadAll(file); err != nil { - return []byte{}, err - } - - return content, nil -} - func redirectOutput(stdout io.ReadCloser) { scanner := bufio.NewScanner(stdout) scanner.Split(bufio.ScanLines) diff --git a/pkg/testcases/context_test.go b/pkg/testcases/context_test.go deleted file mode 100644 index 5de4817..0000000 --- a/pkg/testcases/context_test.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testcases - -import ( - "testing" - - "sigs.k8s.io/windows-operational-readiness/pkg/flags" -) - -func TestConfigurationValidation(t *testing.T) { - var tests = []struct { - answerWant bool - categories flags.ArrayFlags - category string - }{ - { - answerWant: false, - categories: flags.ArrayFlags{"category1", "category2"}, - category: "category3", - }, - { - answerWant: true, - categories: flags.ArrayFlags{"category1", "category2"}, - category: "category2", - }, - } - - for _, tt := range tests { - // Save the category field globally - ctx := TestContext{Categories: tt.categories} - answer := ctx.CategoryEnabled(Category(tt.category)) - - if answer != tt.answerWant { - t.Errorf("got %t, want %t", answer, tt.answerWant) - } - } -} diff --git a/tests/samples/junit_1101_fail.xml b/tests/samples/junit_1101_fail.xml new file mode 100644 index 0000000..68ae5c4 --- /dev/null +++ b/tests/samples/junit_1101_fail.xml @@ -0,0 +1,8 @@ + + + + + > Enter [BeforeEach] [sig-windows] [Feature:Windows] Density [Serial] [Slow] Density [Serial] [Slow] - dump namespaces | framework.go:218 @ 12/10/23 18:35:52.337 (14.239s) > Enter [DeferCleanup (Each)] [sig-windows] [Feature:Windows] Density [Serial] [Slow] - tear down framework | framework.go:215 @ 12/10/23 18:35:52.337 STEP: Destroying namespace "density-test-windows-2761" for this suite. - test/e2e/framework/framework.go:360 @ 12/10/23 18:35:52.337 < Exit [DeferCleanup (Each)] [sig-windows] [Feature:Windows] Density [Serial] [Slow] - tear down framework | framework.go:215 @ 12/10/23 18:35:52.589 (252ms) > Enter [ReportAfterEach] TOP-LEVEL - test/e2e/e2e_test.go:149 @ 12/10/23 18:35:52.589 < Exit [ReportAfterEach] TOP-LEVEL - test/e2e/e2e_test.go:149 @ 12/10/23 18:35:52.589 (0s) + + + \ No newline at end of file diff --git a/tests/samples/junit_1101_valid.xml b/tests/samples/junit_1101_valid.xml new file mode 100644 index 0000000..640da79 --- /dev/null +++ b/tests/samples/junit_1101_valid.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/tests/testcases/context.go b/tests/testcases/context.go new file mode 100644 index 0000000..53687fe --- /dev/null +++ b/tests/testcases/context.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testcases + +import ( + . "github.com/onsi/ginkgo/v2" //nolint + . "github.com/onsi/gomega" //nolint + "sigs.k8s.io/windows-operational-readiness/pkg/flags" + "sigs.k8s.io/windows-operational-readiness/pkg/testcases" +) + +var _ = Describe("Test Context", func() { + Describe("Validating category", func() { + Context("with specific categories in the flag", func() { + It("should enable only those categories", func() { + categories := flags.ArrayFlags{"category1", "category2"} + testCtx := testcases.TestContext{Categories: categories} + + Expect(testCtx.CategoryEnabled("category3")).To(BeFalse()) + Expect(testCtx.CategoryEnabled("category2")).To(BeTrue()) + Expect(testCtx.CategoryEnabled("category1")).To(BeTrue()) + }) + }) + }) +}) diff --git a/tests/windows_operational_readiness_suite_test.go b/tests/windows_operational_readiness_suite_test.go new file mode 100644 index 0000000..631afb2 --- /dev/null +++ b/tests/windows_operational_readiness_suite_test.go @@ -0,0 +1,15 @@ +package main_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" //nolint + . "github.com/onsi/gomega" //nolint + _ "sigs.k8s.io/windows-operational-readiness/tests/testcases" + _ "sigs.k8s.io/windows-operational-readiness/tests/xml" +) + +func TestWindowsOperationalReadiness(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "WindowsOperationalReadiness Suite") +} diff --git a/tests/xml/xml.go b/tests/xml/xml.go new file mode 100644 index 0000000..d03673e --- /dev/null +++ b/tests/xml/xml.go @@ -0,0 +1,53 @@ +package xml + +import ( + . "github.com/onsi/ginkgo/v2" //nolint + . "github.com/onsi/gomega" //nolint + "sigs.k8s.io/windows-operational-readiness/pkg/report" +) + +var _ = Describe("Xml", func() { + + Describe("Must be parsed from junit", func() { + Context("when passed", func() { + It("must have one or more testcases", func() { + var file = "./samples/junit_1101_valid.xml" + r, err := report.UnmarshalXMLFile(file) + Expect(err).To(BeNil()) + + Expect(r.Category).To(Equal("Core.Concurrent")) + + for _, suite := range r.Suites { + Expect(suite.Name).To(Equal("Kubernetes e2e suite")) + Expect(suite.Tests).To(Equal("7391")) + Expect(suite.Time).To(Equal("22.44")) + Expect(len(suite.TestCases)).To(BeNumerically(">=", 1)) + } + }) + }) + + Context("when failed", func() { + It("must render the error", func() { + var file = "./samples/junit_1101_fail.xml" + r, err := report.UnmarshalXMLFile(file) + Expect(err).To(BeNil()) + + Expect(r.Category).To(Equal("Core.Concurrent")) + Expect(r.Failures).To(Equal("1")) + + for _, suite := range r.Suites { + Expect(suite.Name).To(Equal("Kubernetes e2e suite")) + Expect(suite.Tests).To(Equal("7391")) + Expect(suite.Time).To(Equal("34.75")) + Expect(len(suite.Failures)).To(Equal(1)) + Expect(len(suite.TestCases)).To(BeNumerically(">=", 1)) + for _, test := range suite.TestCases { + Expect(len(test.Failure)).To(Equal(1)) + Expect(test.Status).To(Equal(report.StatusFailed)) + } + } + }) + }) + }) + +})