Skip to content

Commit

Permalink
Merge pull request #104 from knabben/clean-category
Browse files Browse the repository at this point in the history
Adding reporter command line
  • Loading branch information
knabben authored Dec 11, 2023
2 parents 8554e5c + 926dbb3 commit a7b7f87
Show file tree
Hide file tree
Showing 17 changed files with 452 additions and 201 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,14 @@ jobs:
with:
go-version: 1.21
- uses: actions/checkout@v3
- name: Run unit-tests
- name: Run unit tests
run: |
go test -v ./...
- name: Run bdd tests
run: |
go install github.com/onsi/ginkgo/v2/ginkgo
go get github.com/onsi/gomega/...
make test
- name: Build the binary
run: |
make build
Expand Down
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ build: ## Build the binary using local golang
./hack/build_k8s_test_binary.sh ${KUBERNETES_HASH}
go build -o ./op-readiness .

.PHONY: test
test: ## Run the Sonobuoy plugin
pushd tests; ginkgo run -v; popd


## --------------------------------------
## Container
## --------------------------------------
Expand Down
81 changes: 81 additions & 0 deletions cmd/reporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
"fmt"
"log"
"strconv"
"strings"

"github.com/spf13/cobra"
"sigs.k8s.io/windows-operational-readiness/pkg/report"
)

var (
directory string
failed int
passed int
)

var reporterCmd = &cobra.Command{
Use: "reporter",
Short: "Parse XML Junit results and print a columnar output",
Long: `Parse XML Junit results and print a columnar output`,
Run: func(cmd *cobra.Command, args []string) {
suites, err := report.ParseXMLFiles(directory)
if err != nil {
log.Fatal(err)
}
for _, n := range suites {
presentTestSuite(n)
}
fmt.Printf("Passed tests: %d\n", passed)
fmt.Printf("Failed tests: %d\n", failed)
},
}

func init() {
reporterCmd.Flags().StringVar(&directory, "dir", "", "directory to search in for xml files")
}

func presentTestSuite(r *report.TestSuites) {
for _, t := range r.Suites {
if len(t.TestCases) == 0 {
ftime, err := strconv.ParseFloat(t.Time, 32)
if err != nil {
continue
}
fmt.Printf("[FAILED] | %s - %s | (%2.2fs) | %s\n", t.Index, r.Category, ftime, r.Name)
failed++
}

for _, c := range t.TestCases {
ftime, err := strconv.ParseFloat(c.Time, 32)
if err != nil {
continue
}
fmt.Printf("[%s] | %s - %s | (%2.2fs) | %s | %s\n", strings.ToUpper(string(c.Status)), t.Index, r.Category, ftime, r.Name, c.Name)
if c.Status == report.StatusPassed {
passed++
}
if c.Status == report.StatusFailed {
failed++
}
}
}
}
60 changes: 38 additions & 22 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,30 @@ package cmd
import (
"fmt"
"os"
"path"

"github.com/spf13/cobra"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/windows-operational-readiness/pkg/flags"
"sigs.k8s.io/windows-operational-readiness/pkg/report"
"sigs.k8s.io/windows-operational-readiness/pkg/testcases"
)

func init() {
rootCmd.PersistentFlags().StringVar(&testDirectory, "test-directory", "", "Path to YAML root directory containing the tests.")
rootCmd.PersistentFlags().StringVar(&E2EBinary, "e2e-binary", "./e2e.test", "The E2E Ginkgo default binary used to run the tests.")
rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, etc.)")
rootCmd.PersistentFlags().StringVar(&kubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
rootCmd.PersistentFlags().StringVar(&reportDir, "report-dir", getEnvOrDefault("ARTIFACTS", ""), "Report dump directory, uses artifact for CI integration when set.")
rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Do not run actual tests, used for sanity check.")
rootCmd.PersistentFlags().BoolVar(&verbose, "verbose", false, "Enable Ginkgo verbosity.")
rootCmd.PersistentFlags().Var(&categories, "category", "Append category of tests you want to run, default empty will run all tests.")

rootCmd.AddCommand(reporterCmd)
}

// NewLoggerConfig return the configuration object for the logger.
func NewLoggerConfig(options ...zap.Option) *zap.Logger {
core := zapcore.NewCore(zapcore.NewConsoleEncoder(zapcore.EncoderConfig{
Expand Down Expand Up @@ -64,7 +79,6 @@ var (
os.Exit(1)
}
testCtx := testcases.NewTestContext(E2EBinary, kubeConfig, provider, specs, dryRun, verbose, reportDir, categories)

targetedSpecs := make([]testcases.Specification, 0)
for _, s := range specs {
if !testCtx.CategoryEnabled(s.Category) {
Expand All @@ -74,18 +88,31 @@ var (
}
}

for sIdx, s := range targetedSpecs {
zap.L().Info(fmt.Sprintf("[OpReadinessTests] %d / %d Specifications - Running %d Test(s) for Ops Readiness specification: %v", sIdx+1, len(targetedSpecs), len(s.TestCases), s.Category))
for specIdx, s := range targetedSpecs {
zap.L().Info(fmt.Sprintf("[OpReadinessTests] %d / %d Specifications - Running %d Test(s) for Ops Readiness specification: %v", specIdx+1, len(targetedSpecs), len(s.TestCases), s.Category))

if len(s.TestCases) == 0 {
zap.L().Info(fmt.Sprintf("[%s] No Operational Readiness tests to run", string(s.Category)))
} else {
for tIdx, t := range s.TestCases {
logPrefix := fmt.Sprintf("[%s] %v / %v Tests - ", s.Category, tIdx+1, len(s.TestCases))
zap.L().Info(fmt.Sprintf("%sRunning Operational Readiness Test: %v", logPrefix, t.Description))
if err = t.RunTest(testCtx, tIdx+1); err != nil {
zap.L().Error(fmt.Sprintf("%sFailed Operational Readiness Test: %v, error is %v", logPrefix, t.Description, zap.Error(err)))
} else {
zap.L().Info(fmt.Sprintf("%sPassed Operational Readiness Test: %v", logPrefix, t.Description))
continue
}

for testIdx, t := range s.TestCases {
prefix := fmt.Sprintf("%d%d", specIdx+1, testIdx+1)
logPrefix := fmt.Sprintf("[%s] %v / %v Tests - ", s.Category, testIdx+1, len(s.TestCases))
zap.L().Info(fmt.Sprintf("%sRunning Operational Readiness Test: %v", logPrefix, t.Description))
if err = t.RunTest(testCtx, prefix); err != nil {
zap.L().Error(fmt.Sprintf("%sFailed Operational Readiness Test: %v, error is %v", logPrefix, t.Description, zap.Error(err)))
} else {
zap.L().Info(fmt.Sprintf("%sPassed Operational Readiness Test: %v", logPrefix, t.Description))
}

// Specs already ran, cleaning up the XML Junit report
if testCtx.ReportDir != "" && !testCtx.DryRun {
fileName := "junit_" + prefix + "01.xml"
junitReport := path.Join(testCtx.ReportDir, fileName)
zap.L().Info("Cleaning XML files", zap.String("file", fileName), zap.String("path", junitReport))
if err := report.CleanupJUnitXML(junitReport, string(s.Category), t.Description, testIdx+1); err != nil {
zap.L().Error(err.Error())
}
}
}
Expand All @@ -105,14 +132,3 @@ func getEnvOrDefault(key, defaultString string) string {
}
return defaultString
}

func init() {
rootCmd.PersistentFlags().StringVar(&testDirectory, "test-directory", "", "Path to YAML root directory containing the tests.")
rootCmd.PersistentFlags().StringVar(&E2EBinary, "e2e-binary", "./e2e.test", "The E2E Ginkgo default binary used to run the tests.")
rootCmd.PersistentFlags().StringVar(&provider, "provider", "local", "The name of the Kubernetes provider (gce, gke, aws, local, skeleton, etc.)")
rootCmd.PersistentFlags().StringVar(&kubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
rootCmd.PersistentFlags().StringVar(&reportDir, "report-dir", getEnvOrDefault("ARTIFACTS", ""), "Report dump directory, uses artifact for CI integration when set.")
rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Do not run actual tests, used for sanity check.")
rootCmd.PersistentFlags().BoolVar(&verbose, "verbose", false, "Enable Ginkgo verbosity.")
rootCmd.PersistentFlags().Var(&categories, "category", "Append category of tests you want to run, default empty will run all tests.")
}
25 changes: 15 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,25 @@ go 1.21

require (
github.com/go-playground/validator/v10 v10.10.0
github.com/onsi/ginkgo/v2 v2.13.2
github.com/onsi/gomega v1.29.0
github.com/spf13/cobra v1.4.0
go.uber.org/zap v1.20.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gopkg.in/yaml.v3 v3.0.1
k8s.io/client-go v0.23.3
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
github.com/imdario/mergo v0.3.5 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand All @@ -28,15 +32,16 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/term v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.14.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/apimachinery v0.23.3 // indirect
Expand Down
Loading

0 comments on commit a7b7f87

Please sign in to comment.