diff --git a/Dockerfile b/Dockerfile index bcb5408..278a9b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,3 +19,17 @@ RUN pip3 install wheel && \ pip3 install --no-cache-dir -r /workspace/oc-pyang-repo/requirements.txt && \ pip3 install --no-cache-dir -r /workspace/pyangbind-repo/requirements.txt # pip3 install pyangbind + +RUN apt install -y npm && npm install -g npm && npm install -g badge-maker + +# Downloading gcloud package +RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz + +# Installing the package +RUN mkdir -p /usr/local/gcloud \ + && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ + && /usr/local/gcloud/google-cloud-sdk/install.sh + +# Adding the package path to local +ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin + diff --git a/README.md b/README.md index 697ae51..92415e0 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,16 @@ To run this CI tool on GCB for a GitHub project, the [GCB App](https://github.com/marketplace/google-cloud-build) needs to be enabled for the target OpenConfig models repo. +## Posting Status Badges + +This is done through a code path in `post_results` that generates an +`upload-badge.sh` file if the CI was triggered on a master branch push. The +badge is created using the +[badge-maker](https://www.npmjs.com/package/badge-maker) package used by +[shields.io](https://shields.io/), whose output svg file is then uploaded to +cloud storage and made public. The script also sets the no-cache option to avoid +GitHub from excessively caching the badge. + ## Future Improvements A custom build container image would, diff --git a/cmd_gen/main.go b/cmd_gen/main.go index 8247b34..66af861 100644 --- a/cmd_gen/main.go +++ b/cmd_gen/main.go @@ -33,6 +33,7 @@ var ( modelRoot string // modelRoot is the root directory of the models. repoSlug string // repoSlug is the "owner/repo" name of the models repo (e.g. openconfig/public). commitSHA string + branchName string // branchName is the name of the branch where the commit occurred. prNumber int compatReports string // e.g. "goyang-ygot,pyangbind,pyang@1.7.8" extraPyangVersions string // e.g. "1.2.3,3.4.5" @@ -68,6 +69,7 @@ func init() { flag.StringVar(&repoSlug, "repo-slug", "openconfig/public", "repo where CI is run") flag.StringVar(&commitSHA, "commit-sha", "", "commit SHA of the PR") flag.IntVar(&prNumber, "pr-number", 0, "PR number") + flag.StringVar(&branchName, "branch", "", "branch name of commit") flag.StringVar(&compatReports, "compat-report", "", "comma-separated validators (e.g. goyang-ygot,pyang@1.7.8,pyang@head) in compatibility report instead of a standalone PR status") flag.StringVar(&skippedValidators, "skipped-validators", "", "comma-separated validators (e.g. goyang-ygot,pyang@1.7.8,pyang@head) not to be ran at all, not even in the compatibility report") flag.StringVar(&extraPyangVersions, "extra-pyang-versions", "", "comma-separated extra pyang versions to run") @@ -225,7 +227,9 @@ func genOpenConfigValidatorScript(g labelPoster, validatorId, version string, mo for _, modelDirName := range modelDirNames { if disabledModelPaths[modelDirName] { log.Printf("skipping disabled model directory %s", modelDirName) - g.PostLabel("skipped: "+modelDirName, commonci.LabelColors["orange"], owner, repo, prNumber) + if prNumber != 0 { + g.PostLabel("skipped: "+modelDirName, commonci.LabelColors["orange"], owner, repo, prNumber) + } continue } cmdStr, err := genValidatorCommandForModelDir(validatorId, resultsDir, modelDirName, modelMap) @@ -301,15 +305,31 @@ func main() { log.Fatalf("modelDirName and validator can only be specified for local cmd generation") } + badgeOnly := false + // If it's a push on master, just upload badge for normal validators as the only action. + if prNumber == 0 { + if branchName != "master" { + log.Fatalf("cmd_gen: There is no action to take for a non-master branch push, please re-examine your push triggers") + } + badgeOnly = true + } + + // Skip testing non-widely used validators, as we don't need to post badges for those tools. + if badgeOnly { + for validatorId, validator := range commonci.Validators { + if !validator.IsWidelyUsedTool { + // Here we assume simply that non widely-used checks don't have a version specified. + skippedValidators += "," + validatorId + } + } + } + repoSplit := strings.Split(repoSlug, "/") owner = repoSplit[0] repo = repoSplit[1] if commitSHA == "" { log.Fatalf("no commit SHA") } - if prNumber == 0 { - log.Fatalf("no PR number") - } h, err := commonci.NewGitHubRequestHandler() if err != nil { @@ -324,9 +344,11 @@ func main() { } compatReports = commonci.ValidatorAndVersionsDiff(compatReports, skippedValidators) - // Notify later CI steps of the validators that should be reported as a compatibility report. - if err := ioutil.WriteFile(commonci.CompatReportValidatorsFile, []byte(compatReports), 0444); err != nil { - log.Fatalf("error while writing compatibility report validators file %q: %v", commonci.CompatReportValidatorsFile, err) + if !badgeOnly { + // Notify later CI steps of the validators that should be reported as a compatibility report. + if err := ioutil.WriteFile(commonci.CompatReportValidatorsFile, []byte(compatReports), 0444); err != nil { + log.Fatalf("error while writing compatibility report validators file %q: %v", commonci.CompatReportValidatorsFile, err) + } } _, compatValidatorsMap := commonci.GetValidatorAndVersionsFromString(compatReports) @@ -367,7 +389,7 @@ func main() { } // Post initial PR status. - if !compatValidatorsMap[validatorId][version] { + if !badgeOnly && !compatValidatorsMap[validatorId][version] { if errs := postInitialStatus(h, validatorId, version); errs != nil { log.Fatal(errs) } diff --git a/cmd_gen/main_test.go b/cmd_gen/main_test.go index 947cd2a..2d6cc42 100644 --- a/cmd_gen/main_test.go +++ b/cmd_gen/main_test.go @@ -33,6 +33,7 @@ func (p *postLabelRecorder) PostLabel(labelName, labelColor, owner, repo string, } func TestGenOpenConfigLinterScript(t *testing.T) { + prNumber = 1 basicModelMap, err := commonci.ParseOCModels("testdata") if err != nil { t.Fatalf("TestGenOpenConfigLinterScript: Failed to parse models for testing: %v", err) diff --git a/commonci/commonci.go b/commonci/commonci.go index 47f64d3..dc12dfc 100644 --- a/commonci/commonci.go +++ b/commonci/commonci.go @@ -50,6 +50,9 @@ const ( OutFileName = "out" // FailFileName by convention contains the stderr of the script file. FailFileName = "fail" + // BadgeUploadCmdFile is output by post_results to upload the correct + // status badge to GCS. + BadgeUploadCmdFile = "upload-badge.sh" ) // AppendVersionToName appends the version to the given validator name @@ -82,6 +85,9 @@ type Validator struct { // ReportOnly indicates that it's not itself a validator, it's just a // CI item that does reporting on other validators. ReportOnly bool + // IsWidelyUsedTool indicates that the tool is a widely used tool whose + // status should be reported on the front page of the repository. + IsWidelyUsedTool bool } // StatusName determines the status description for the version of the validator. @@ -97,28 +103,33 @@ var ( // The key is a unique identifier that's safe to use as a directory name. Validators = map[string]*Validator{ "pyang": &Validator{ - Name: "pyang", - IsPerModel: true, + Name: "pyang", + IsPerModel: true, + IsWidelyUsedTool: true, }, "oc-pyang": &Validator{ Name: "OpenConfig Linter", IsPerModel: true, }, "pyangbind": &Validator{ - Name: "pyangbind", - IsPerModel: true, + Name: "pyangbind", + IsPerModel: true, + IsWidelyUsedTool: true, }, "goyang-ygot": &Validator{ - Name: "goyang/ygot", - IsPerModel: true, + Name: "goyang/ygot", + IsPerModel: true, + IsWidelyUsedTool: true, }, "yanglint": &Validator{ - Name: "yanglint", - IsPerModel: true, + Name: "yanglint", + IsPerModel: true, + IsWidelyUsedTool: true, }, "confd": &Validator{ - Name: "ConfD Basic", - IsPerModel: true, + Name: "ConfD Basic", + IsPerModel: true, + IsWidelyUsedTool: true, }, "regexp": &Validator{ Name: "regexp tests", @@ -252,13 +263,16 @@ func GetValidatorAndVersionsFromString(validatorsAndVersionsStr string) ([]Valid if len(vvSegments) == 2 { vv.Version = vvSegments[1] } - compatValidators = append(compatValidators, vv) m, ok := compatValidatorsMap[vv.ValidatorId] if !ok { m = map[string]bool{} compatValidatorsMap[vv.ValidatorId] = m } - m[vv.Version] = true + // De-dup validator@version names. + if !m[vv.Version] { + compatValidators = append(compatValidators, vv) + m[vv.Version] = true + } } return compatValidators, compatValidatorsMap } diff --git a/commonci/commonci_test.go b/commonci/commonci_test.go index 49638d6..9065acb 100644 --- a/commonci/commonci_test.go +++ b/commonci/commonci_test.go @@ -157,8 +157,8 @@ func TestGetValidatorAndVersionsFromString(t *testing.T) { wantVVList: []ValidatorAndVersion{{ValidatorId: "pyang", Version: "1.7.2"}}, wantVVMap: map[string]map[string]bool{"pyang": map[string]bool{"1.7.2": true}}, }, { - desc: "more than one version", - inStr: "pyang@1.7.2,pyang,oc-pyang,pyang@head", + desc: "more than one version with an extraneous comma", + inStr: "pyang@1.7.2,,pyang,oc-pyang,pyang@head", wantVVList: []ValidatorAndVersion{ {ValidatorId: "pyang", Version: "1.7.2"}, {ValidatorId: "pyang", Version: ""}, @@ -176,8 +176,8 @@ func TestGetValidatorAndVersionsFromString(t *testing.T) { }, }, }, { - desc: "more than one version with ending comma", - inStr: "pyang@1.7.2,pyang,oc-pyang,pyang@head,", + desc: "more than one version with ending comma and with a duplicate", + inStr: "pyang@1.7.2,pyang,oc-pyang,pyang@1.7.2,pyang@head,", wantVVList: []ValidatorAndVersion{ {ValidatorId: "pyang", Version: "1.7.2"}, {ValidatorId: "pyang", Version: ""}, @@ -241,8 +241,8 @@ func TestValidatorAndVersionsDiff(t *testing.T) { inBStr: "pyang@1.7.2", wantStr: "", }, { - desc: "more than one version", - inAStr: "pyang@1.7.2,pyang,oc-pyang,pyang@head", + desc: "more than one version and with an extraneous comma", + inAStr: "pyang@1.7.2,pyang,oc-pyang,,pyang@head", inBStr: "pyang@head", wantStr: "pyang@1.7.2,pyang,oc-pyang", }, { @@ -250,6 +250,11 @@ func TestValidatorAndVersionsDiff(t *testing.T) { inAStr: "pyang@1.7.2,pyang,oc-pyang,pyang@head", inBStr: "oc-pyang,pyang@1.7.2", wantStr: "pyang,pyang@head", + }, { + desc: "more than one version deletes with duplicates", + inAStr: "pyang@1.7.2,pyang,oc-pyang,pyang@head,oc-pyang,goyang-ygot", + inBStr: "pyang@1.7.2,goyang-ygot,oc-pyang,pyang@1.7.2", + wantStr: "pyang,pyang@head", }, { desc: "more than one version deletes with deleting something not there", inAStr: "pyang@1.7.2,pyang,oc-pyang,pyang@head", diff --git a/post_results/main.go b/post_results/main.go index 17c5af3..a66d9da 100644 --- a/post_results/main.go +++ b/post_results/main.go @@ -22,6 +22,7 @@ import ( "path/filepath" "sort" "strings" + "text/template" "log" @@ -35,8 +36,8 @@ import ( const ( // The title of the results uses the relevant emoji to show whether it // succeeded or failed. - mdPassSymbol = ":white_check_mark:" - mdFailSymbol = ":no_entry:" + mdPassSymbol = "✅" + mdFailSymbol = "⛔" // IgnorePyangWarnings ignores all warnings from pyang or pyang-based tools. IgnorePyangWarnings = true // IgnoreConfdWarnings ignores all warnings from ConfD. @@ -45,25 +46,53 @@ const ( var ( // flags - validatorId string // validatorId is the unique name identifying the validator (see commonci for all of them) - modelRoot string // modelRoot is the root directory of the models. - repoSlug string // repoSlug is the "owner/repo" name of the models repo (e.g. openconfig/public). - prNumber int - prBranchName string - commitSHA string - version string // version is a specific version of the validator that's being run (empty means latest). + validatorId string // validatorId is the unique name identifying the validator (see commonci for all of them) + modelRoot string // modelRoot is the root directory of the models. + repoSlug string // repoSlug is the "owner/repo" name of the models repo (e.g. openconfig/public). + prNumber int + branchName string // branchName is the name of the branch where the commit occurred. + commitSHA string + version string // version is a specific version of the validator that's being run (empty means latest). // derived flags owner string repo string + + // badgeCmdTemplate is the badge creation and upload command generated for pushes to the master branch. + badgeCmdTemplate = mustTemplate("badgeCmd", `REMOTE_PATH_PFX=gs://artifacts.disco-idea-817.appspot.com/compatibility-badges/{{ .RepoPrefix }}: +RESULTSDIR={{ .ResultsDir }} +upload-public-file() { + gsutil cp $RESULTSDIR/$1 "$REMOTE_PATH_PFX"$1 + gsutil acl ch -u AllUsers:R "$REMOTE_PATH_PFX"$1 + gsutil setmeta -h "Cache-Control:no-cache" "$REMOTE_PATH_PFX"$1 +} +badge "{{ .Status }}" "{{ .ValidatorDesc }}" :{{ .Colour }} > $RESULTSDIR/{{ .ValidatorAndVersion }}.svg +upload-public-file {{ .ValidatorAndVersion }}.svg +upload-public-file {{ .ValidatorAndVersion }}.html +`) ) +// mustTemplate generates a template.Template for a particular named source template +func mustTemplate(name, src string) *template.Template { + return template.Must(template.New(name).Parse(src)) +} + +// badgeCmdParams is the input to the badge template. +type badgeCmdParams struct { + RepoPrefix string + Status string + ValidatorAndVersion string + ValidatorDesc string + Colour string + ResultsDir string +} + func init() { flag.StringVar(&validatorId, "validator", "", "unique name of the validator") flag.StringVar(&modelRoot, "modelRoot", "", "root directory to OpenConfig models") flag.StringVar(&repoSlug, "repo-slug", "", "repo where CI is run") flag.IntVar(&prNumber, "pr-number", 0, "PR number") - flag.StringVar(&prBranchName, "pr-branch", "", "branch name of PR") + flag.StringVar(&branchName, "branch", "", "branch name of commit") flag.StringVar(&commitSHA, "commit-sha", "", "commit SHA of the PR") flag.StringVar(&version, "version", "", "(optional) specific version of the validator tool.") } @@ -335,7 +364,8 @@ func processStandardOutput(rawOut string, pass, noWarnings bool) (string, error) // parseModelResultsHTML transforms the output files of the validator script into HTML // to be displayed on GitHub. -func parseModelResultsHTML(validatorId, validatorResultDir string) (string, bool, error) { +// If condensed=true, then only errors are provided. +func parseModelResultsHTML(validatorId, validatorResultDir string, condensed bool) (string, bool, error) { var htmlOut, modelHTML strings.Builder var prevModelDirName string @@ -356,7 +386,9 @@ func parseModelResultsHTML(validatorId, validatorResultDir string) (string, bool // Write results one modelDir at a time in order to report overall modelDir status. if prevModelDirName != "" && modelDirName != prevModelDirName { - htmlOut.WriteString(sprintSummaryHTML(modelDirPass, prevModelDirName, modelHTML.String())) + if !condensed || !modelDirPass { + htmlOut.WriteString(sprintSummaryHTML(modelDirPass, prevModelDirName, modelHTML.String())) + } modelHTML.Reset() modelDirPass = true } @@ -398,7 +430,9 @@ func parseModelResultsHTML(validatorId, validatorResultDir string) (string, bool return fmt.Errorf("error encountered while processing output for validator %q: %v", validatorId, err) } - modelHTML.WriteString(sprintSummaryHTML(modelPass, modelName, outString)) + if !condensed || !modelPass { + modelHTML.WriteString(sprintSummaryHTML(modelPass, modelName, outString)) + } } return nil }); err != nil { @@ -406,7 +440,9 @@ func parseModelResultsHTML(validatorId, validatorResultDir string) (string, bool } // Edge case: handle last modelDir. - htmlOut.WriteString(sprintSummaryHTML(modelDirPass, prevModelDirName, modelHTML.String())) + if !condensed || !modelDirPass { + htmlOut.WriteString(sprintSummaryHTML(modelDirPass, prevModelDirName, modelHTML.String())) + } return htmlOut.String(), allPass, nil } @@ -414,7 +450,8 @@ func parseModelResultsHTML(validatorId, validatorResultDir string) (string, bool // getResult parses the results for the given validator and its results // directory, and returns the string to be put in a GitHub gist comment as well // as the status (i.e. pass or fail). -func getResult(validatorId, resultsDir string) (string, bool, error) { +// If condensed=true, then only errors are provided. +func getResult(validatorId, resultsDir string, condensed bool) (string, bool, error) { validator, ok := commonci.Validators[validatorId] if !ok { return "", false, fmt.Errorf("validator %q not found!", validatorId) @@ -444,7 +481,10 @@ func getResult(validatorId, resultsDir string) (string, bool, error) { case validator.IsPerModel && validatorId == "misc-checks": outString, pass, err = processMiscChecksOutput(resultsDir) case validator.IsPerModel: - outString, pass, err = parseModelResultsHTML(validatorId, resultsDir) + outString, pass, err = parseModelResultsHTML(validatorId, resultsDir, condensed) + if pass && condensed { + outString = "All passed.\n" + outString + } default: outString = "Test passed." pass = true @@ -453,6 +493,31 @@ func getResult(validatorId, resultsDir string) (string, bool, error) { return outString, pass, err } +// WriteBadgeUploadCmdFile writes a bash script into resultsDir that posts a +// status badge for the given validator and result into cloud storage. +func WriteBadgeUploadCmdFile(validatorDesc, validatorUniqueStr string, pass bool, resultsDir string) (string, error) { + // Badge creation and upload command. + var builder strings.Builder + status := "fail" + colour := "red" + if pass { + status = "pass" + colour = "brightgreen" + } + if err := badgeCmdTemplate.Execute(&builder, &badgeCmdParams{ + RepoPrefix: strings.ReplaceAll(repoSlug, "/", "-"), // Make repo slug safe for use as file name. + Status: status, + ValidatorAndVersion: validatorUniqueStr, + ValidatorDesc: validatorDesc, + Colour: colour, + ResultsDir: resultsDir, + }); err != nil { + return "", err + } + + return builder.String(), nil +} + // getGistHeading gets the description and content of the result gist for the // given validator from its script output file. The "description" is the title // of the gist, and "content" is the script execution output. @@ -542,14 +607,13 @@ func postCompatibilityReport(validatorAndVersions []commonci.ValidatorAndVersion resultsDir := commonci.ValidatorResultsDir(vv.ValidatorId, vv.Version) // Post parsed test results as a gist comment. - testResultString, pass, err := getResult(vv.ValidatorId, resultsDir) + testResultString, pass, err := getResult(vv.ValidatorId, resultsDir, false) if err != nil { return fmt.Errorf("postResult: couldn't parse results for <%s>@<%s> in resultsDir %q: %v", vv.ValidatorId, vv.Version, resultsDir, err) } gistTitle := fmt.Sprintf("%s %s", lintSymbol(pass), validatorDescs[i]) - gistContent := testResultString - id, err := g.AddGistComment(gistID, gistTitle, gistContent) + id, err := g.AddGistComment(gistID, gistTitle, testResultString) if err != nil { fmt.Errorf("postResult: could not add gist comment: %v", err) } @@ -570,51 +634,90 @@ func postResult(validatorId, version string) error { if !ok { return fmt.Errorf("postResult: validator %q not found!", validatorId) } + resultsDir := commonci.ValidatorResultsDir(validatorId, version) - var url, gistID string - var err error - var g *commonci.GithubRequestHandler + pushToMaster := false + // If it's a push on master, just upload badge for normal validators as the only action. + if prNumber == 0 { + if branchName != "master" { + return fmt.Errorf("postResult: There is no action to take for a non-master branch push, please re-examine your push triggers") + } + pushToMaster = true + } + + if !pushToMaster { + compatReportsStr, err := readFile(commonci.CompatReportValidatorsFile) + if err != nil { + return fmt.Errorf("postResult: %v", err) + } + compatValidators, compatValidatorsMap := commonci.GetValidatorAndVersionsFromString(compatReportsStr) - compatReportsStr, err := readFile(commonci.CompatReportValidatorsFile) + if validatorId == "compat-report" { + log.Printf("Processing compatibility report for %s", compatReportsStr) + return postCompatibilityReport(compatValidators) + } + + // Skip PR status reporting if validator is part of compatibility report. + if compatValidatorsMap[validatorId][version] { + log.Printf("Validator %s part of compatibility report, skipping reporting standalone PR status.", commonci.AppendVersionToName(validatorId, version)) + return nil + } + } + + // Get information needed for posting badge or GitHub gist. + validatorDesc, runOutput, err := getGistHeading(validatorId, version, resultsDir) if err != nil { return fmt.Errorf("postResult: %v", err) } - compatValidators, compatValidatorsMap := commonci.GetValidatorAndVersionsFromString(compatReportsStr) - - if validatorId == "compat-report" { - log.Printf("Processing compatibility report for %s", compatReportsStr) - return postCompatibilityReport(compatValidators) + testResultString, pass, err := getResult(validatorId, resultsDir, false) + if err != nil { + return fmt.Errorf("postResult: couldn't parse results: %v", err) } - // Skip reporting if validator is part of compatibility report. - if compatValidatorsMap[validatorId][version] { - log.Printf("Validator %s part of compatibility report, skipping reporting standalone PR status.", commonci.AppendVersionToName(validatorId, version)) + if pushToMaster { + if validator.ReportOnly { + // Only upload results for running validators. + return nil + } + // Output badge creation & upload commands into a file to be executed. + validatorUniqueStr := commonci.AppendVersionToName(validatorId, version) + uploadCmdFileContent, err := WriteBadgeUploadCmdFile(validatorDesc, validatorUniqueStr, pass, resultsDir) + if err != nil { + return fmt.Errorf("postResult: couldn't upload badge command for <%s>@<%s> in resultsDir %q: %v", validatorId, version, resultsDir, err) + } + badgeUploadFile := filepath.Join(resultsDir, commonci.BadgeUploadCmdFile) + if err := ioutil.WriteFile(badgeUploadFile, []byte(uploadCmdFileContent), 0444); err != nil { + log.Fatalf("error while writing validator pass file %q: %v", badgeUploadFile, err) + return err + } + + // Put output into a file to be uploaded and linked by the badges. + outputHTML := fmt.Sprintf("

%s

Execution output:\n%s

", testResultString, runOutput) + outputFile := filepath.Join(resultsDir, validatorUniqueStr+".html") + if err := ioutil.WriteFile(outputFile, []byte(outputHTML), 0666); err != nil { + log.Fatalf("error while writing output file %q: %v", outputFile, err) + return err + } return nil } - resultsDir := commonci.ValidatorResultsDir(validatorId, version) + + var url, gistID string + var g *commonci.GithubRequestHandler // Create gist representing test results. The "validatorDesc" is the - // title of the gist, and "content" is the script execution output. - validatorDesc, content, err := getGistHeading(validatorId, version, resultsDir) - if err != nil { - return fmt.Errorf("postResult: %v", err) - } + // title of the gist, and "runOutput" is the script execution output. if err := commonci.Retry(5, "CreateCIOutputGist", func() error { g, err = commonci.NewGitHubRequestHandler() if err != nil { return err } - url, gistID, err = g.CreateCIOutputGist(validatorDesc, content) + url, gistID, err = g.CreateCIOutputGist(validatorDesc, runOutput) return err }); err != nil { return fmt.Errorf("postResult: couldn't create gist: %v", err) } // Post parsed test results as a gist comment. - testResultString, pass, err := getResult(validatorId, resultsDir) - if err != nil { - return fmt.Errorf("postResult: couldn't parse results: %v", err) - } if _, err := g.AddGistComment(gistID, fmt.Sprintf("%s %s", lintSymbol(pass), validatorDesc), testResultString); err != nil { fmt.Errorf("postResult: could not add gist comment: %v", err) } @@ -651,8 +754,8 @@ func main() { if commitSHA == "" { log.Fatalf("no commit SHA") } - if prBranchName == "" { - log.Fatalf("no PR branch name supplied") + if prNumber == 0 && branchName != "master" { + log.Fatalf("no PR branch name supplied or push trigger not on master branch") } if err := postResult(validatorId, version); err != nil { diff --git a/post_results/main_test.go b/post_results/main_test.go index 8ea8efc..fba553d 100644 --- a/post_results/main_test.go +++ b/post_results/main_test.go @@ -15,6 +15,7 @@ package main import ( + "fmt" "strings" "testing" @@ -165,6 +166,8 @@ func TestGetResult(t *testing.T) { inValidatorId string wantPass bool wantOut string + wantCondensedOut string + wantCondensedOutSame bool wantErrSubstr string }{{ name: "basic pyang pass", @@ -172,26 +175,28 @@ func TestGetResult(t *testing.T) { inValidatorId: "oc-pyang", wantPass: true, wantOut: `
- :white_check_mark: acl + ✅ acl
- :white_check_mark: openconfig-acl + ✅ openconfig-acl Passed.
- :white_check_mark: optical-transport + ✅ optical-transport
- :white_check_mark: openconfig-optical-amplifier + ✅ openconfig-optical-amplifier Passed.
- :white_check_mark: openconfig-transport-line-protection + ✅ openconfig-transport-line-protection Passed.
+`, + wantCondensedOut: `All passed. `, }, { name: "pyang with an empty fail file", @@ -200,30 +205,33 @@ Passed. wantPass: false, wantOut: `Validator script failed -- infra bug? Test failed with no stderr output.`, + wantCondensedOutSame: true, }, { name: "basic non-pyang pass", inValidatorResultDir: "testdata/oc-pyang", inValidatorId: "goyang-ygot", wantPass: true, wantOut: `
- :white_check_mark: acl + ✅ acl
- :white_check_mark: openconfig-acl + ✅ openconfig-acl Passed.
- :white_check_mark: optical-transport + ✅ optical-transport
- :white_check_mark: openconfig-optical-amplifier + ✅ openconfig-optical-amplifier Passed.
- :white_check_mark: openconfig-transport-line-protection + ✅ openconfig-transport-line-protection Passed. warning foo
+`, + wantCondensedOut: `All passed. `, }, { name: "pyang with pass and fails", @@ -231,28 +239,45 @@ warning foo
inValidatorId: "pyang", wantPass: false, wantOut: `
- :no_entry: acl + ⛔ acl
- :no_entry: openconfig-acl + ⛔ openconfig-acl
- :no_entry: optical-transport + ⛔ optical-transport
- :no_entry: openconfig-optical-amplifier + ⛔ openconfig-optical-amplifier Failed.
- :white_check_mark: openconfig-transport-line-protection + ✅ openconfig-transport-line-protection Passed.
+`, + wantCondensedOut: `
+ ⛔ acl +
+ ⛔ openconfig-acl + +
+
+
+ ⛔ optical-transport +
+ ⛔ openconfig-optical-amplifier +Failed. +
+
`, }, { name: "non-pyang with pass and fails", @@ -260,24 +285,39 @@ Passed. inValidatorId: "yanglint", wantPass: false, wantOut: `
- :no_entry: acl + ⛔ acl
- :no_entry: openconfig-acl + ⛔ openconfig-acl /workspace/release/yang/wifi/mac/openconfig-wifi-mac.yang:1244: error: enum value "B" should be of the form UPPERCASE_WITH_UNDERSCORES: B
- :no_entry: optical-transport + ⛔ optical-transport
- :no_entry: openconfig-optical-amplifier + ⛔ openconfig-optical-amplifier Failed.
- :white_check_mark: openconfig-transport-line-protection + ✅ openconfig-transport-line-protection Passed. warning foo
+`, + wantCondensedOut: `
+ ⛔ acl +
+ ⛔ openconfig-acl +/workspace/release/yang/wifi/mac/openconfig-wifi-mac.yang:1244: error: enum value "B" should be of the form UPPERCASE_WITH_UNDERSCORES: B
+
+
+
+ ⛔ optical-transport +
+ ⛔ openconfig-optical-amplifier +Failed. +
+
`, }, { name: "non-per-model pass -- no fail file", @@ -285,78 +325,90 @@ warning foo
inValidatorId: "regexp", wantPass: true, wantOut: `Test passed.`, + wantCondensedOutSame: true, }, { name: "non-per-model fail -- empty fail file", inValidatorResultDir: "testdata/regexp-tests2", inValidatorId: "regexp", wantPass: false, wantOut: `Test failed with no stderr output.`, + wantCondensedOutSame: true, }, { name: "non-per-model fail", inValidatorResultDir: "testdata/regexp-tests-fail", inValidatorId: "regexp", wantPass: false, wantOut: "I failed\n", + wantCondensedOutSame: true, }, { name: "pyang script fail", inValidatorResultDir: "testdata/oc-pyang-script-fail", inValidatorId: "oc-pyang", wantPass: false, wantOut: "Validator script failed -- infra bug?\nI failed\n", + wantCondensedOutSame: true, }, { name: "openconfig-version, revision version, and .spec.yml checks all pass", inValidatorResultDir: "testdata/misc-checks-pass", inValidatorId: "misc-checks", wantPass: true, wantOut: `
- :white_check_mark: openconfig-version update check + ✅ openconfig-version update check 4 file(s) correctly updated.
- :white_check_mark: .spec.yml build reachability check + ✅ .spec.yml build reachability check 8 files reached by build rules.
`, + wantCondensedOutSame: true, }, { name: "openconfig-version, revision version, and .spec.yml checks all fail", inValidatorResultDir: "testdata/misc-checks-fail", inValidatorId: "misc-checks", wantPass: false, wantOut: `
- :no_entry: openconfig-version update check + ⛔ openconfig-version update check
  • changed-version-to-noversion.yang: openconfig-version was removed
  • openconfig-acl.yang: file updated but PR version not updated: "1.2.2"
    • - :no_entry: .spec.yml build reachability check + ⛔ .spec.yml build reachability check
  • changed-noversion-to-unreached.yang: file not used by any .spec.yml build.
  • changed-unreached-to-unreached.yang: file not used by any .spec.yml build.
  • changed-version-to-unreached.yang: file not used by any .spec.yml build.
  • unchanged-unreached.yang: file not used by any .spec.yml build.
    • `, + wantCondensedOutSame: true, }} for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotOut, gotPass, err := getResult(tt.inValidatorId, tt.inValidatorResultDir) - if err != nil { - if diff := errdiff.Substring(err, tt.wantErrSubstr); diff != "" { - t.Fatalf("did not get expected error, %s", diff) + for _, condensed := range []bool{false, true} { + t.Run(fmt.Sprintf(tt.name+"@condensed=%v", condensed), func(t *testing.T) { + gotOut, gotPass, err := getResult(tt.inValidatorId, tt.inValidatorResultDir, condensed) + if err != nil { + if diff := errdiff.Substring(err, tt.wantErrSubstr); diff != "" { + t.Fatalf("did not get expected error, %s", diff) + } + return } - return - } - if gotPass != tt.wantPass { - t.Errorf("gotPass %v, want %v", gotPass, tt.wantPass) - } - if diff := cmp.Diff(strings.Split(tt.wantOut, "\n"), strings.Split(gotOut, "\n")); diff != "" { - t.Errorf("(-want, +got):\n%s", diff) - } - }) + if gotPass != tt.wantPass { + t.Errorf("gotPass %v, want %v", gotPass, tt.wantPass) + } + wantOut := tt.wantOut + if condensed && !tt.wantCondensedOutSame { + wantOut = tt.wantCondensedOut + } + if diff := cmp.Diff(strings.Split(wantOut, "\n"), strings.Split(gotOut, "\n")); diff != "" { + t.Errorf("(-want, +got):\n%s", diff) + } + }) + } } } -func TestGetGistInfo(t *testing.T) { +func TestGetGistHeading(t *testing.T) { tests := []struct { name string inValidatorResultDir string @@ -413,3 +465,63 @@ func TestGetGistInfo(t *testing.T) { }) } } + +func TestWriteBadgeUploadCmdFile(t *testing.T) { + repoSlug = "openconfig/repo" + tests := []struct { + name string + inValidatorDesc string + inValidatorUniqueStr string + inVersion string + inPass bool + inResultsDir string + wantFileContent string + }{{ + name: "pass", + inValidatorDesc: "pyang@1.2.3", + inValidatorUniqueStr: "pyang@latest", + inPass: true, + inResultsDir: "results-dir", + wantFileContent: `REMOTE_PATH_PFX=gs://artifacts.disco-idea-817.appspot.com/compatibility-badges/openconfig-repo: +RESULTSDIR=results-dir +upload-public-file() { + gsutil cp $RESULTSDIR/$1 "$REMOTE_PATH_PFX"$1 + gsutil acl ch -u AllUsers:R "$REMOTE_PATH_PFX"$1 + gsutil setmeta -h "Cache-Control:no-cache" "$REMOTE_PATH_PFX"$1 +} +badge "pass" "pyang@1.2.3" :brightgreen > $RESULTSDIR/pyang@latest.svg +upload-public-file pyang@latest.svg +upload-public-file pyang@latest.html +`, + }, { + name: "fail", + inValidatorDesc: "pyang@2.3.4", + inValidatorUniqueStr: "pyang", + inPass: false, + inResultsDir: "results-directory", + wantFileContent: `REMOTE_PATH_PFX=gs://artifacts.disco-idea-817.appspot.com/compatibility-badges/openconfig-repo: +RESULTSDIR=results-directory +upload-public-file() { + gsutil cp $RESULTSDIR/$1 "$REMOTE_PATH_PFX"$1 + gsutil acl ch -u AllUsers:R "$REMOTE_PATH_PFX"$1 + gsutil setmeta -h "Cache-Control:no-cache" "$REMOTE_PATH_PFX"$1 +} +badge "fail" "pyang@2.3.4" :red > $RESULTSDIR/pyang.svg +upload-public-file pyang.svg +upload-public-file pyang.html +`, + }} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := WriteBadgeUploadCmdFile(tt.inValidatorDesc, tt.inValidatorUniqueStr, tt.inPass, tt.inResultsDir) + if err != nil { + t.Fatal(err) + } + + if got != tt.wantFileContent { + t.Errorf("gotFileContent:\n%v\nwant:\n%v", got, tt.wantFileContent) + } + }) + } +} diff --git a/validators/compat_report.sh b/validators/compat_report.sh index 8a2f05b..a54f44b 100755 --- a/validators/compat_report.sh +++ b/validators/compat_report.sh @@ -3,11 +3,12 @@ ROOT_DIR=/workspace USERCONFIG_DIR=$ROOT_DIR/user-config -echo validators to be put in compability report: -cat $USERCONFIG_DIR/compat-report-validators.txt - if ! [ -s $USERCONFIG_DIR/compat-report-validators.txt ]; then + echo "skipping: no validators to report in compatibility report" exit 0 fi -$GOPATH/bin/post_results -validator=compat-report -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA -pr-number=$_PR_NUMBER +echo validators to be put in compability report: +cat $USERCONFIG_DIR/compat-report-validators.txt + +$GOPATH/bin/post_results -validator=compat-report -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -commit-sha=$COMMIT_SHA -pr-number=$_PR_NUMBER -branch=$BRANCH_NAME diff --git a/validators/confd/test.sh b/validators/confd/test.sh index 3f62849..b4b1864 100755 --- a/validators/confd/test.sh +++ b/validators/confd/test.sh @@ -22,4 +22,8 @@ if bash $RESULTSDIR/script.sh $CONFDC $CONFDPATH > $OUTFILE 2> $FAILFILE; then # Delete fail file if it's empty and the script passed. find $FAILFILE -size 0 -delete fi -$GOPATH/bin/post_results -validator=confd -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=confd -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME +BADGEFILE=$RESULTSDIR/upload-badge.sh +if stat $BADGEFILE; then + bash $BADGEFILE +fi diff --git a/validators/goyang-ygot/test.sh b/validators/goyang-ygot/test.sh index ec737ea..db6cef2 100755 --- a/validators/goyang-ygot/test.sh +++ b/validators/goyang-ygot/test.sh @@ -14,4 +14,8 @@ if bash $RESULTSDIR/script.sh > $OUTFILE 2> $FAILFILE; then # Delete fail file if it's empty and the script passed. find $FAILFILE -size 0 -delete fi -$GOPATH/bin/post_results -validator=goyang-ygot -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=goyang-ygot -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME +BADGEFILE=$RESULTSDIR/upload-badge.sh +if stat $BADGEFILE; then + bash $BADGEFILE +fi diff --git a/validators/misc-checks/test.sh b/validators/misc-checks/test.sh index afc9414..b2a9111 100755 --- a/validators/misc-checks/test.sh +++ b/validators/misc-checks/test.sh @@ -32,7 +32,7 @@ cat $RESULTSDIR/*.pr-file-parse-log > $RESULTSDIR/pr-file-parse-log 2>> $OUTFILE # changed-files.txt REPODIR=$RESULTSDIR/base_repo -git clone -b $_HEAD_BRANCH "git@github.com:$_REPO_SLUG.git" $REPODIR +git clone -b $BRANCH_NAME "git@github.com:$_REPO_SLUG.git" $REPODIR cd $REPODIR BASE_COMMIT=$(git merge-base $COMMIT_SHA origin/master) git diff --name-only $BASE_COMMIT | grep -E '.*\.yang$' > $RESULTSDIR/changed-files.txt 2>> $OUTFILE @@ -44,4 +44,4 @@ if find $REPODIR -name '*.yang' | xargs $GOPATH/bin/ocversion -p $REPODIR > $RES find $FAILFILE -size 0 -delete fi -$GOPATH/bin/post_results -validator=misc-checks -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=misc-checks -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME diff --git a/validators/oc-pyang/test.sh b/validators/oc-pyang/test.sh index 9c2649b..1131788 100755 --- a/validators/oc-pyang/test.sh +++ b/validators/oc-pyang/test.sh @@ -52,7 +52,7 @@ if bash $RESULTSDIR/script.sh $VENVDIR/bin/pyang --plugindir $OCPYANG_PLUGIN_DIR # Delete fail file if it's empty and the script passed. find $FAILFILE -size 0 -delete fi -$GOPATH/bin/post_results -validator=oc-pyang -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=oc-pyang -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME ########################## CLEANUP ############################# teardown diff --git a/validators/pyang/test.sh b/validators/pyang/test.sh index bf3cb93..2d1160a 100755 --- a/validators/pyang/test.sh +++ b/validators/pyang/test.sh @@ -26,7 +26,11 @@ run-pyang-version() { # Delete fail file if it's empty and the script passed. find $RESULTSDIR/$FAILFILE_NAME -size 0 -delete fi - $GOPATH/bin/post_results -validator=pyang -version=$1 -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA + $GOPATH/bin/post_results -validator=pyang -version=$1 -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME + BADGEFILE=$RESULTSDIR/upload-badge.sh + if stat $BADGEFILE; then + bash $BADGEFILE + fi } run-pyang-head() { @@ -49,7 +53,7 @@ run-pyang-head() { # Delete fail file if it's empty and the script passed. find $RESULTSDIR/$FAILFILE_NAME -size 0 -delete fi - $GOPATH/bin/post_results -validator=pyang -version="head" -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA + $GOPATH/bin/post_results -validator=pyang -version="head" -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME } run-pyang-head & @@ -69,7 +73,11 @@ if bash $RESULTSDIR/script.sh $VENVDIR/bin/pyang > $RESULTSDIR/$OUTFILE_NAME 2> # Delete fail file if it's empty and the script passed. find $RESULTSDIR/$FAILFILE_NAME -size 0 -delete fi -$GOPATH/bin/post_results -validator=pyang -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=pyang -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME +BADGEFILE=$RESULTSDIR/upload-badge.sh +if stat $BADGEFILE; then + bash $BADGEFILE +fi ########################## CLEANUP ############################# wait diff --git a/validators/pyangbind/test.sh b/validators/pyangbind/test.sh index 3a4a202..4159e9b 100755 --- a/validators/pyangbind/test.sh +++ b/validators/pyangbind/test.sh @@ -40,7 +40,11 @@ if bash $RESULTSDIR/script.sh $VENVDIR/bin/pyang --plugindir $PYANGBIND_PLUGIN_D # Delete fail file if it's empty and the script passed. find $FAILFILE -size 0 -delete fi -$GOPATH/bin/post_results -validator=pyangbind -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=pyangbind -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME +BADGEFILE=$RESULTSDIR/upload-badge.sh +if stat $BADGEFILE; then + bash $BADGEFILE +fi ########################## CLEANUP ############################# teardown diff --git a/validators/regexp/test.sh b/validators/regexp/test.sh index 45aff22..3f38ac6 100755 --- a/validators/regexp/test.sh +++ b/validators/regexp/test.sh @@ -13,4 +13,4 @@ if go test -v gotests/regexp > $OUTFILE 2> $FAILFILE; then # Delete fail file if it's empty and the script passed. find $FAILFILE -size 0 -delete fi -$GOPATH/bin/post_results -validator=regexp -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=regexp -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME diff --git a/validators/yanglint/test.sh b/validators/yanglint/test.sh index f693e7f..8d701e7 100755 --- a/validators/yanglint/test.sh +++ b/validators/yanglint/test.sh @@ -17,4 +17,8 @@ if bash $RESULTSDIR/script.sh > $OUTFILE 2> $FAILFILE; then # Delete fail file if it's empty and the script passed. find $FAILFILE -size 0 -delete fi -$GOPATH/bin/post_results -validator=yanglint -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-branch=$_HEAD_BRANCH -commit-sha=$COMMIT_SHA +$GOPATH/bin/post_results -validator=yanglint -modelRoot=$_MODEL_ROOT -repo-slug=$_REPO_SLUG -pr-number=$_PR_NUMBER -commit-sha=$COMMIT_SHA -branch=$BRANCH_NAME +BADGEFILE=$RESULTSDIR/upload-badge.sh +if stat $BADGEFILE; then + bash $BADGEFILE +fi