diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 61939ed..d5cdae0 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
name: Go
on:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..e5c3b28
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,29 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement (CLA). You (or your employer) retain the copyright to your
+contribution; this simply gives us permission to use and redistribute your
+contributions as part of the project. Head over to
+ to see your current agreements on file or
+to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code Reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google/conduct/).
diff --git a/Dockerfile b/Dockerfile
index 278a9b7..82ec5b2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,18 @@
-FROM gcr.io/cloud-builders/go:debian
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang
SHELL ["/bin/bash", "-c"]
#RUN git clone https://github.com/mbj4668/pyang.git /workspace/results/pyang@head/pyang
diff --git a/LICENSE b/LICENSE
index 8dada3e..d645695 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,3 +1,4 @@
+
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@@ -178,7 +179,7 @@
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "{}"
+ boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
@@ -186,7 +187,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
- Copyright {yyyy} {name of copyright owner}
+ Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/README.md b/README.md
index a9af731..8d5b55a 100644
--- a/README.md
+++ b/README.md
@@ -171,7 +171,7 @@ regexp | Files moved into GOPATH from its folder during CI build
pyang & pyangbind | pip
oc-pyang | git clone
goyang/ygot | go get
-yanglint | Debian package periodically uploaded to cloud storage
+yanglint | Debian packages (libyang2 and libyang2-tools) periodically uploaded to cloud storage. These are renamed libyang.deb and yanglint.deb respectively in the GCS bucket.
## Setting Up GCB
diff --git a/cmd_gen/main.go b/cmd_gen/main.go
index ee506e2..9c768af 100644
--- a/cmd_gen/main.go
+++ b/cmd_gen/main.go
@@ -122,7 +122,7 @@ var (
// GCB script, which together create the running environment for the
// generated validator script.
scriptTemplates = map[string]*scriptSpec{
- "pyang": &scriptSpec{
+ "pyang": {
headerTemplate: mustTemplate("pyang-header", `#!/bin/bash
workdir={{ .ResultsDir }}
mkdir -p "$workdir"
@@ -138,7 +138,7 @@ script_options=(
function run-dir() {
declare prefix="$workdir"/"$1"=="$2"==
shift 2
- echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+ echo pyang "${options[@]}" "$@" > ${prefix}cmd
if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
mv ${prefix}pass ${prefix}fail
fi
@@ -147,25 +147,27 @@ function run-dir() {
perModelTemplate: mustTemplate("pyang", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
`),
},
- "oc-pyang": &scriptSpec{
+ "oc-pyang": {
headerTemplate: mustTemplate("oc-pyang-header", `#!/bin/bash
workdir={{ .ResultsDir }}
mkdir -p "$workdir"
`+"{{`"+util.PYANG_MSG_TEMPLATE_STRING+"`}}"+`
cmd="$@"
options=(
- -p {{ .ModelRoot }}
- -p {{ .RepoRoot }}/third_party/ietf
--openconfig
--ignore-error=OC_RELATIVE_PATH
+ -p {{ .ModelRoot }}
+ -p {{ .RepoRoot }}/third_party/ietf
)
script_options=(
--msg-template "$PYANG_MSG_TEMPLATE"
)
function run-dir() {
declare prefix="$workdir"/"$1"=="$2"==
+ local cmd_display_options=( --plugindir '$OCPYANG_PLUGIN_DIR' "${options[@]}" )
+ local options=( --plugindir "$OCPYANG_PLUGIN_DIR" "${options[@]}" )
shift 2
- echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+ echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
mv ${prefix}pass ${prefix}fail
fi
@@ -174,26 +176,33 @@ function run-dir() {
perModelTemplate: mustTemplate("oc-pyang", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
`),
},
- "pyangbind": &scriptSpec{
+ "pyangbind": {
headerTemplate: mustTemplate("pyangbind-header", `#!/bin/bash
workdir={{ .ResultsDir }}
mkdir -p "$workdir"
`+"{{`"+util.PYANG_MSG_TEMPLATE_STRING+"`}}"+`
cmd="$@"
options=(
+ -f pybind
-p {{ .ModelRoot }}
-p {{ .RepoRoot }}/third_party/ietf
- -f pybind
)
script_options=(
--msg-template "$PYANG_MSG_TEMPLATE"
)
function run-dir() {
declare prefix="$workdir"/"$1"=="$2"==
- local options=( -o "$1"."$2".binding.py "${options[@]}" )
+ local output_file="$1"."$2".binding.py
+ local cmd_display_options=( --plugindir '$PYANGBIND_PLUGIN_DIR' -o "${output_file}" "${options[@]}" )
+ local options=( --plugindir "$PYANGBIND_PLUGIN_DIR" -o "${output_file}" "${options[@]}" )
shift 2
- echo $cmd "${options[@]}" "$@" > ${prefix}cmd
- if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
+ echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
+ status=0
+ $cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
+ if [[ $status -eq "0" ]]; then
+ python "${output_file}" &>> ${prefix}pass || status=1
+ fi
+ if [[ $status -eq "1" ]]; then
mv ${prefix}pass ${prefix}fail
fi
}
@@ -201,7 +210,7 @@ function run-dir() {
perModelTemplate: mustTemplate("pyangbind", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
`),
},
- "goyang-ygot": &scriptSpec{
+ "goyang-ygot": {
headerTemplate: mustTemplate("goyang-ygot-header", `#!/bin/bash
workdir={{ .ResultsDir }}
mkdir -p "$workdir"
@@ -226,8 +235,9 @@ function run-dir() {
status=0
$cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
cd "$outdir"
- go get &>> ${prefix}pass || status=1
if [[ $status -eq "0" ]]; then
+ go mod init &>> ${prefix}pass || status=1
+ go mod tidy &>> ${prefix}pass || status=1
go build &>> ${prefix}pass || status=1
fi
if [[ $status -eq "1" ]]; then
@@ -238,7 +248,7 @@ function run-dir() {
perModelTemplate: mustTemplate("goyang-ygot", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
`),
},
- "yanglint": &scriptSpec{
+ "yanglint": {
headerTemplate: mustTemplate("yanglint-header", `#!/bin/bash
workdir={{ .ResultsDir }}
mkdir -p "$workdir"
@@ -261,7 +271,7 @@ function run-dir() {
perModelTemplate: mustTemplate("yanglint", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
`),
},
- "confd": &scriptSpec{
+ "confd": {
headerTemplate: mustTemplate("confd-header", `#!/bin/bash
workdir={{ .ResultsDir }}
mkdir -p "$workdir"
@@ -275,7 +285,7 @@ if [[ $status -eq "1" ]]; then
fi
`),
},
- "misc-checks": &scriptSpec{
+ "misc-checks": {
headerTemplate: mustTemplate("misc-checks-header", `#!/bin/bash
workdir={{ .ResultsDir }}
mkdir -p "$workdir"
@@ -341,6 +351,7 @@ type labelPoster interface {
// will be run only on a single model as specified in the .spec.yml file.
// 2. Thus, a validation command and result is provided for each model.
// 3. A file indicating pass/fail is output for each model into the given result directory.
+//
// Files names follow the "modelDir==model==status" format with no file extensions.
// The local flag indicates to run this as a helper to generate the script,
// rather than running it within GCB.
@@ -461,7 +472,7 @@ func main() {
// If it's a push on master, just upload badge for normal validators as the only action.
if prNumber == 0 {
if branchName != "master" {
- log.Fatalf("cmd_gen: There is no action to take for a non-master branch push, please re-examine your push triggers")
+ log.Fatalf("cmd_gen: pr-number not supplied as a flag to the build. Try re-running (by commenting \"/gcbrun\" on the GitHub PR) to see whether the $_PR_NUMBER substitution variable for Google Cloud Build gets passed into the build. If this branch is not associated with a PR, then it is inferred that this is a non-master branch push action, and thus there is no CI action that is expected, and in this case please re-examine your push triggers.")
}
pushToMaster = true
}
diff --git a/cmd_gen/main_test.go b/cmd_gen/main_test.go
index 2a30917..1bac902 100644
--- a/cmd_gen/main_test.go
+++ b/cmd_gen/main_test.go
@@ -66,7 +66,7 @@ script_options=(
function run-dir() {
declare prefix="$workdir"/"$1"=="$2"==
shift 2
- echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+ echo pyang "${options[@]}" "$@" > ${prefix}cmd
if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
mv ${prefix}pass ${prefix}fail
fi
@@ -97,7 +97,7 @@ script_options=(
function run-dir() {
declare prefix="$workdir"/"$1"=="$2"==
shift 2
- echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+ echo pyang "${options[@]}" "$@" > ${prefix}cmd
if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
mv ${prefix}pass ${prefix}fail
fi
@@ -116,18 +116,20 @@ mkdir -p "$workdir"
PYANG_MSG_TEMPLATE='messages:{{path:"{file}" line:{line} code:"{code}" type:"{type}" level:{level} message:'"'{msg}'}}"
cmd="$@"
options=(
- -p testdata
- -p /workspace/third_party/ietf
--openconfig
--ignore-error=OC_RELATIVE_PATH
+ -p testdata
+ -p /workspace/third_party/ietf
)
script_options=(
--msg-template "$PYANG_MSG_TEMPLATE"
)
function run-dir() {
declare prefix="$workdir"/"$1"=="$2"==
+ local cmd_display_options=( --plugindir '$OCPYANG_PLUGIN_DIR' "${options[@]}" )
+ local options=( --plugindir "$OCPYANG_PLUGIN_DIR" "${options[@]}" )
shift 2
- echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+ echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
mv ${prefix}pass ${prefix}fail
fi
@@ -147,19 +149,26 @@ mkdir -p "$workdir"
PYANG_MSG_TEMPLATE='messages:{{path:"{file}" line:{line} code:"{code}" type:"{type}" level:{level} message:'"'{msg}'}}"
cmd="$@"
options=(
+ -f pybind
-p testdata
-p /workspace/third_party/ietf
- -f pybind
)
script_options=(
--msg-template "$PYANG_MSG_TEMPLATE"
)
function run-dir() {
declare prefix="$workdir"/"$1"=="$2"==
- local options=( -o "$1"."$2".binding.py "${options[@]}" )
+ local output_file="$1"."$2".binding.py
+ local cmd_display_options=( --plugindir '$PYANGBIND_PLUGIN_DIR' -o "${output_file}" "${options[@]}" )
+ local options=( --plugindir "$PYANGBIND_PLUGIN_DIR" -o "${output_file}" "${options[@]}" )
shift 2
- echo $cmd "${options[@]}" "$@" > ${prefix}cmd
- if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
+ echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
+ status=0
+ $cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
+ if [[ $status -eq "0" ]]; then
+ python "${output_file}" &>> ${prefix}pass || status=1
+ fi
+ if [[ $status -eq "1" ]]; then
mv ${prefix}pass ${prefix}fail
fi
}
@@ -196,8 +205,9 @@ function run-dir() {
status=0
$cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
cd "$outdir"
- go get &>> ${prefix}pass || status=1
if [[ $status -eq "0" ]]; then
+ go mod init &>> ${prefix}pass || status=1
+ go mod tidy &>> ${prefix}pass || status=1
go build &>> ${prefix}pass || status=1
fi
if [[ $status -eq "1" ]]; then
diff --git a/cmd_gen/testdata/acl/.spec.yml b/cmd_gen/testdata/acl/.spec.yml
index 0539d97..09783fe 100644
--- a/cmd_gen/testdata/acl/.spec.yml
+++ b/cmd_gen/testdata/acl/.spec.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
- name: openconfig-acl
docs:
- yang/acl/openconfig-packet-match-types.yang
diff --git a/cmd_gen/testdata/optical-transport/.spec.yml b/cmd_gen/testdata/optical-transport/.spec.yml
index 1e4c8bf..ab07ba4 100644
--- a/cmd_gen/testdata/optical-transport/.spec.yml
+++ b/cmd_gen/testdata/optical-transport/.spec.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
- name: openconfig-terminal-device
docs:
- yang/optical-transport/openconfig-transport-types.yang
diff --git a/commonci/github.go b/commonci/github.go
index fcaedf9..05ef633 100644
--- a/commonci/github.go
+++ b/commonci/github.go
@@ -184,7 +184,7 @@ func (g *GithubRequestHandler) IsPRApproved(owner, repo string, prNumber int) (b
}
// PostLabel posts the given label to the PR. It is idempotent.
-// unit tests can be created based onon actual models-ci repo data that's sent back.
+// unit tests can be created based on actual models-ci repo data that's sent back.
func (g *GithubRequestHandler) PostLabel(labelName, labelColor, owner, repo string, prNumber int) error {
if g.labels[labelName] {
// Label already exists.
@@ -249,13 +249,18 @@ func (g *GithubRequestHandler) AddPRComment(body *string, owner, repo string, pr
return nil
}
-// AddOrEditPRComment posts or edits a comment to a PR.
+// AddEditOrDeletePRComment posts or edits a comment to a PR.
// If signature is empty, it's equivalent to AddPRComment.
+// Signature is a string that can be found in the comment in order to find and edit it.
// Otherwise, all comments are searched first, and if any matches the
// signature, then the comment is edited. If none matches, then a new comment
// is posted.
-func (g *GithubRequestHandler) AddOrEditPRComment(signature string, body *string, owner, repo string, prNumber int) error {
+// If body is nil, then it indicates delete.
+func (g *GithubRequestHandler) AddEditOrDeletePRComment(signature string, body *string, owner, repo string, prNumber int) error {
if signature == "" {
+ if body == nil {
+ return fmt.Errorf("PR comment body unspecified")
+ }
return g.AddPRComment(body, owner, repo, prNumber)
}
@@ -269,20 +274,37 @@ func (g *GithubRequestHandler) AddOrEditPRComment(signature string, body *string
return err
}); err != nil {
// If somehow this fails, we should be resilient and just post another comment.
+ if body == nil {
+ return fmt.Errorf("list comments failed -- cannot find comment to delete")
+ }
return g.AddPRComment(body, owner, repo, prNumber)
}
for _, pc := range comments {
if strings.Contains(*pc.Body, signature) {
- if err := Retry(5, "edit PR comment", func() error {
- _, _, err := g.client.Issues.EditComment(ctx, owner, repo, *pc.ID, &github.IssueComment{Body: body})
- return err
- }); err != nil {
- return g.AddPRComment(body, owner, repo, prNumber)
+ switch body {
+ case nil:
+ if err := Retry(5, "delete PR comment", func() error {
+ _, err := g.client.Issues.DeleteComment(ctx, owner, repo, *pc.ID)
+ return err
+ }); err != nil {
+ return fmt.Errorf("cannot delete comment: %v", err)
+ }
+ default:
+ if err := Retry(5, "edit PR comment", func() error {
+ _, _, err := g.client.Issues.EditComment(ctx, owner, repo, *pc.ID, &github.IssueComment{Body: body})
+ return err
+ }); err != nil {
+ return g.AddPRComment(body, owner, repo, prNumber)
+ }
}
return nil
}
}
+
+ if body == nil {
+ return fmt.Errorf("PR comment body unspecified")
+ }
return g.AddPRComment(body, owner, repo, prNumber)
}
diff --git a/commonci/testdata/acl/.spec.yml b/commonci/testdata/acl/.spec.yml
index 0539d97..09783fe 100644
--- a/commonci/testdata/acl/.spec.yml
+++ b/commonci/testdata/acl/.spec.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
- name: openconfig-acl
docs:
- yang/acl/openconfig-packet-match-types.yang
diff --git a/commonci/testdata/optical-transport/.spec.yml b/commonci/testdata/optical-transport/.spec.yml
index 1e4c8bf..ab07ba4 100644
--- a/commonci/testdata/optical-transport/.spec.yml
+++ b/commonci/testdata/optical-transport/.spec.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
- name: openconfig-terminal-device
docs:
- yang/optical-transport/openconfig-transport-types.yang
diff --git a/go.mod b/go.mod
index 94b7f12..05cabe2 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,7 @@ require (
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-querystring v1.0.0 // indirect
github.com/openconfig/gnmi v0.0.0-20200508230933-d19cebf5e7be
- github.com/openconfig/goyang v0.2.1
+ github.com/openconfig/goyang v1.0.0
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
google.golang.org/protobuf v1.21.0
gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86
diff --git a/go.sum b/go.sum
index af44c8e..18f7e5a 100644
--- a/go.sum
+++ b/go.sum
@@ -35,8 +35,8 @@ github.com/openconfig/gnmi v0.0.0-20200414194230-1597cc0f2600/go.mod h1:M/EcuapN
github.com/openconfig/gnmi v0.0.0-20200508230933-d19cebf5e7be h1:VEK8utxoyZu/hkpjLxvuBmK5yW3NmBo/v/Wu5VQAJVs=
github.com/openconfig/gnmi v0.0.0-20200508230933-d19cebf5e7be/go.mod h1:M/EcuapNQgvzxo1DDXHK4tx3QpYM/uG4l591v33jG2A=
github.com/openconfig/goyang v0.0.0-20200115183954-d0a48929f0ea/go.mod h1:dhXaV0JgHJzdrHi2l+w0fZrwArtXL7jEFoiqLEdmkvU=
-github.com/openconfig/goyang v0.2.1 h1:TaADsLGzULGQL8xTmvLjmXqZ/J1F73dMzURDqrv9cLs=
-github.com/openconfig/goyang v0.2.1/go.mod h1:vX61x01Q46AzbZUzG617vWqh/cB+aisc+RrNkXRd3W8=
+github.com/openconfig/goyang v1.0.0 h1:nYaFu7BOAk/eQn4CgAUjgYPfp3J6CdXrBryp32E5CjI=
+github.com/openconfig/goyang v1.0.0/go.mod h1:vX61x01Q46AzbZUzG617vWqh/cB+aisc+RrNkXRd3W8=
github.com/openconfig/ygot v0.6.0/go.mod h1:o30svNf7O0xK+R35tlx95odkDmZWS9JyWWQSmIhqwAs=
github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
diff --git a/post_results/main.go b/post_results/main.go
index c05cff3..3d40d45 100644
--- a/post_results/main.go
+++ b/post_results/main.go
@@ -41,6 +41,8 @@ const (
IgnorePyangWarnings = true
// IgnoreConfdWarnings ignores all warnings from ConfD.
IgnoreConfdWarnings = false
+ // bucketName is the Google storage bucket name.
+ bucketName = "openconfig"
)
var (
@@ -59,7 +61,7 @@ var (
prNumber int
// badgeCmdTemplate is the badge creation and upload command generated for pushes to the master branch.
- badgeCmdTemplate = mustTemplate("badgeCmd", `REMOTE_PATH_PFX=gs://artifacts.disco-idea-817.appspot.com/compatibility-badges/{{ .RepoPrefix }}:
+ badgeCmdTemplate = mustTemplate("badgeCmd", fmt.Sprintf(`REMOTE_PATH_PFX=gs://%s/compatibility-badges/{{ .RepoPrefix }}:
RESULTSDIR={{ .ResultsDir }}
upload-public-file() {
gsutil cp $RESULTSDIR/$1 "$REMOTE_PATH_PFX"$1
@@ -69,7 +71,7 @@ upload-public-file() {
badge "{{ .Status }}" "{{ .ValidatorDesc }}" :{{ .Colour }} > $RESULTSDIR/{{ .ValidatorAndVersion }}.svg
upload-public-file {{ .ValidatorAndVersion }}.svg
upload-public-file {{ .ValidatorAndVersion }}.html
-`)
+`, bucketName))
)
// mustTemplate generates a template.Template for a particular named source template
@@ -193,9 +195,7 @@ func readGoyangVersionsLog(logPath string, masterBranch bool, fileProperties map
value = value[1 : len(value)-1] // Remove enclosing quotes.
}
switch name {
- case "openconfig-version":
- fallthrough
- case "latest-revision-version":
+ case "openconfig-version", "belonging-module", "latest-revision-version":
if masterBranch {
name = "master-" + name
}
@@ -212,101 +212,64 @@ func readGoyangVersionsLog(logPath string, masterBranch bool, fileProperties map
// according to semantic versioning rules.
// Note that any increase is fine, including jumps, e.g. 1.0.0 -> 1.0.2.
// If there isn't an increase, a descriptive error message is returned.
-func checkSemverIncrease(oldVersion, newVersion string) error {
+func checkSemverIncrease(oldVersion, newVersion, versionStringName string) (*semver.Version, *semver.Version, error) {
newV, err := semver.StrictNewVersion(newVersion)
if err != nil {
- return fmt.Errorf("invalid version string: %q", newVersion)
+ return nil, nil, fmt.Errorf("invalid version string: %q", newVersion)
}
oldV, err := semver.StrictNewVersion(oldVersion)
switch {
case err != nil:
- return fmt.Errorf("unexpected error, base branch version string unparseable: %q", oldVersion)
+ return nil, nil, fmt.Errorf("unexpected error, base branch version string unparseable: %q", oldVersion)
case newV.Equal(oldV):
- return fmt.Errorf("file updated but PR version not updated: %q", oldVersion)
+ return nil, nil, fmt.Errorf("file updated but %s string not updated: %q", versionStringName, oldVersion)
case !newV.GreaterThan(oldV):
- return fmt.Errorf("new semantic version not valid, old version: %q, new version: %q", oldVersion, newVersion)
+ return nil, nil, fmt.Errorf("new semantic version not valid, old version: %q, new version: %q", oldVersion, newVersion)
default:
- return nil
+ return oldV, newV, nil
}
}
-// processMiscChecksOutput takes the raw result output from the misc-checks
-// results directory and returns its formatted report and pass/fail status.
-func processMiscChecksOutput(resultsDir string) (string, bool, error) {
- fileProperties := map[string]map[string]string{}
- changedFiles, err := readYangFilesList(filepath.Join(resultsDir, "changed-files.txt"))
- if err != nil {
- return "", false, err
- }
- for _, file := range changedFiles {
- if _, ok := fileProperties[file]; !ok {
- fileProperties[file] = map[string]string{}
- }
- fileProperties[file]["changed"] = "true"
- }
- if err := readGoyangVersionsLog(filepath.Join(resultsDir, "pr-file-parse-log"), false, fileProperties); err != nil {
- return "", false, err
- }
- if err := readGoyangVersionsLog(filepath.Join(resultsDir, "master-file-parse-log"), true, fileProperties); err != nil {
- return "", false, err
- }
-
- var ocVersionViolations []string
- ocVersionChangedCount := 0
- var reachabilityViolations []string
- filesReachedCount := 0
- // Only look at the PR's files as they might be different from the master's files.
- allNonEmptyPRFiles, err := readYangFilesList(filepath.Join(resultsDir, "all-non-empty-files.txt"))
- if err != nil {
- return "", false, err
- }
- for _, file := range allNonEmptyPRFiles {
- properties, ok := fileProperties[file]
+type fileAndVersion struct {
+ name string
+ version *semver.Version
+}
- // Reachability check
- if !ok || properties["reachable"] != "true" {
- reachabilityViolations = append(reachabilityViolations, sprintLineHTML("%s: file not used by any .spec.yml build.", file))
- // If the file was not reached, then its other
- // parameters would not have been parsed by goyang, so
- // simply skip the rest of the checks.
- continue
+// versionGroupViolationsHTML returns the version violations where a group of
+// module/submodule files don't have matching versions.
+func versionGroupViolationsHTML(moduleFileGroups map[string][]fileAndVersion) []string {
+ var violations []string
+
+ var modules []string
+ for m := range moduleFileGroups {
+ modules = append(modules, m)
+ }
+ sort.Strings(modules)
+ for _, moduleName := range modules {
+ latestVersion := semver.MustParse("0.0.0")
+ latestVersionModule := ""
+ for _, nameAndVersion := range moduleFileGroups[moduleName] {
+ if nameAndVersion.version.GreaterThan(latestVersion) {
+ latestVersion = nameAndVersion.version
+ latestVersionModule = nameAndVersion.name
+ }
}
- filesReachedCount += 1
+ latestVersionString := latestVersion.Original()
- // openconfig-version update check
- ocVersion, hasVersion := properties["openconfig-version"]
- masterOcVersion, hadVersion := properties["master-openconfig-version"]
- switch {
- case properties["changed"] != "true":
- // We assume the versioning is correct without change.
- case hadVersion && hasVersion:
- if err := checkSemverIncrease(masterOcVersion, ocVersion); err != nil {
- ocVersionViolations = append(ocVersionViolations, sprintLineHTML(file+": "+err.Error()))
- } else {
- ocVersionChangedCount += 1
+ var violation strings.Builder
+ for _, nameAndVersion := range moduleFileGroups[moduleName] {
+ if version := nameAndVersion.version.Original(); version != latestVersionString {
+ if violation.Len() != 0 {
+ violation.WriteString(",")
+ }
+ violation.WriteString(fmt.Sprintf(" %s (%s)", nameAndVersion.name, version))
}
- case hadVersion && !hasVersion:
- ocVersionViolations = append(ocVersionViolations, sprintLineHTML("%s: openconfig-version was removed", file))
- default: // If didn't have version before, any new version is accepted.
- ocVersionChangedCount += 1
}
- }
-
- // Compute HTML string and pass/fail status.
- var out strings.Builder
- var pass = true
- appendViolationOut := func(desc string, violations []string, passString string) {
- if len(violations) == 0 {
- out.WriteString(sprintSummaryHTML(commonci.BoolStatusToString(true), desc, passString))
- } else {
- out.WriteString(sprintSummaryHTML(commonci.BoolStatusToString(false), desc, strings.Join(violations, "")))
- pass = false
+ if violation.Len() != 0 {
+ violations = append(violations, sprintLineHTML("module set %s is at %s (%s), non-matching files:%s", moduleName, latestVersionString, latestVersionModule, violation.String()))
}
}
- appendViolationOut("openconfig-version update check", ocVersionViolations, fmt.Sprintf("%d file(s) correctly updated.\n", ocVersionChangedCount))
- appendViolationOut(".spec.yml build reachability check", reachabilityViolations, fmt.Sprintf("%d files reached by build rules.\n", filesReachedCount))
-
- return out.String(), pass, nil
+ return violations
}
// processStandardOutput takes raw pyang/confd output and transforms it to an
@@ -391,6 +354,12 @@ func processPyangOutput(rawOut string, pass, noWarnings bool) (string, error) {
return out.String(), nil
}
+// userfyBashCommand changes the bash command displayed to the user to be
+// something that's easier to use.
+func userfyBashCommand(cmd string) string {
+ return strings.NewReplacer("/workspace/", "$OC_WORKSPACE/", "$OCPYANG_PLUGIN_DIR", "$GOPATH/src/github.com/openconfig/oc-pyang/openconfig_pyang/plugins", "$PYANGBIND_PLUGIN_DIR", "$GOPATH/src/github.com/robshakir/pyangbind/pyangbind/plugin").Replace(cmd)
+}
+
// parseModelResultsHTML transforms the output files of the validator script into HTML
// to be displayed on GitHub.
// If condensed=true, then only errors are provided.
@@ -442,7 +411,7 @@ func parseModelResultsHTML(validatorId, validatorResultDir string, condensed boo
// order, ${prefix}cmd should be walked first,
// such that ${prefix}pass or ${prefix}fail
// will have it ready to display to the user.
- bashCommand = outString
+ bashCommand = userfyBashCommand(outString)
bashCommandModelDirName = modelDirName
bashCommandModelName = modelName
return nil
@@ -478,7 +447,7 @@ func parseModelResultsHTML(validatorId, validatorResultDir string, condensed boo
// Display bash command that produced the validator result if it exists.
var bashCommandSummary string
if bashCommand != "" && bashCommandModelDirName == modelDirName && bashCommandModelName == modelName {
- bashCommandSummary = sprintSummaryHTML("cmd", "bash command", "%s
", bashCommand)
+ bashCommandSummary = fmt.Sprintf("%s %s\n%s
\n", commonci.Emoji("cmd"), "bash command", bashCommand)
}
// Also display the error string.
modelHTML.WriteString(sprintSummaryHTML(status, modelName, bashCommandSummary+outString))
@@ -500,17 +469,24 @@ func parseModelResultsHTML(validatorId, validatorResultDir string, condensed boo
// getResult parses the results for the given validator and its results
// directory, and returns the string to be put in a GitHub gist comment as well
// as the status (i.e. pass or fail).
+//
+// It also returns a newline-separated string of the list of all YANG files
+// that have their major versions updated, which is empty if there are no major
+// version changes.
+//
// If condensed=true, then only errors are provided.
-func getResult(validatorId, resultsDir string, condensed bool) (string, bool, error) {
+func getResult(validatorId, resultsDir string, condensed bool) (string, bool, versionRecordSlice, error) {
validator, ok := commonci.Validators[validatorId]
if !ok {
- return "", false, fmt.Errorf("validator %q not found!", validatorId)
+ return "", false, nil, fmt.Errorf("validator %q not found!", validatorId)
}
// outString is parsed stdout.
var outString string
// pass is the overall validation result.
var pass bool
+ // versionRecords contains all information regarding YANG version changes.
+ var versionRecords versionRecordSlice
failFileBytes, err := ioutil.ReadFile(filepath.Join(resultsDir, commonci.FailFileName))
// existent fail file == failure.
@@ -529,7 +505,7 @@ func getResult(validatorId, resultsDir string, condensed bool) (string, bool, er
}
pass = false
case validator.IsPerModel && validatorId == "misc-checks":
- outString, pass, err = processMiscChecksOutput(resultsDir)
+ outString, pass, versionRecords, err = processMiscChecksOutput(resultsDir)
case validator.IsPerModel:
outString, pass, err = parseModelResultsHTML(validatorId, resultsDir, condensed)
if pass && condensed {
@@ -540,7 +516,7 @@ func getResult(validatorId, resultsDir string, condensed bool) (string, bool, er
pass = true
}
- return outString, pass, err
+ return outString, pass, versionRecords, err
}
// WriteBadgeUploadCmdFile writes a bash script into resultsDir that posts a
@@ -657,7 +633,7 @@ func postCompatibilityReport(validatorAndVersions []commonci.ValidatorAndVersion
resultsDir := commonci.ValidatorResultsDir(vv.ValidatorId, vv.Version)
// Post parsed test results as a gist comment.
- testResultString, pass, err := getResult(vv.ValidatorId, resultsDir, false)
+ testResultString, pass, _, err := getResult(vv.ValidatorId, resultsDir, false)
if err != nil {
return fmt.Errorf("postResult: couldn't parse results for <%s>@<%s> in resultsDir %q: %v", vv.ValidatorId, vv.Version, resultsDir, err)
}
@@ -665,18 +641,41 @@ func postCompatibilityReport(validatorAndVersions []commonci.ValidatorAndVersion
gistTitle := fmt.Sprintf("%s %s", commonci.Emoji(commonci.BoolStatusToString(pass)), validatorDescs[i])
id, err := g.AddGistComment(gistID, gistTitle, testResultString)
if err != nil {
- fmt.Errorf("postResult: could not add gist comment: %v", err)
+ return fmt.Errorf("postResult: could not add gist comment: %v", err)
}
commentBuilder.WriteString(fmt.Sprintf("%s [%s](%s#gistcomment-%d)\n", commonci.Emoji(commonci.BoolStatusToString(pass)), validatorDescs[i], gistURL, id))
}
comment := commentBuilder.String()
- if err := g.AddOrEditPRComment("Compatibility Report for commit", &comment, owner, repo, prNumber); err != nil {
+ if err := g.AddEditOrDeletePRComment("Compatibility Report for commit", &comment, owner, repo, prNumber); err != nil {
return fmt.Errorf("postCompatibilityReport: couldn't post comment: %v", err)
}
return nil
}
+// postBreakingChangeLabel posts label and information on whether the PR
+// contains breaking changes that necessitate a repository version bump.
+func postBreakingChangeLabel(g *commonci.GithubRequestHandler, versionRecords versionRecordSlice) error {
+ if versionRecords.hasBreaking() {
+ if err := g.PostLabel("breaking", "FF0000", owner, repo, prNumber); err != nil {
+ return fmt.Errorf("couldn't post label: %v", err)
+ }
+ g.DeleteLabel("non-breaking", owner, repo, prNumber)
+ } else {
+ if err := g.PostLabel("non-breaking", "00FF00", owner, repo, prNumber); err != nil {
+ return fmt.Errorf("couldn't post label: %v", err)
+ }
+ // Don't error out on error since it's possible the label doesn't exist.
+ g.DeleteLabel("breaking", owner, repo, prNumber)
+ }
+ // NOTE: "ajor" is not a typo.
+ majorVersionChangesComment := versionRecords.MajorVersionChanges()
+ if err := g.AddEditOrDeletePRComment("ajor YANG version changes in commit", &majorVersionChangesComment, owner, repo, prNumber); err != nil {
+ return fmt.Errorf("couldn't post major YANG version changes comment: %v", err)
+ }
+ return nil
+}
+
// postResult retrieves the test output for the given validator and version
// from its results folder and posts a gist and PR status linking to the gist.
func postResult(validatorId, version string) error {
@@ -719,7 +718,7 @@ func postResult(validatorId, version string) error {
if err != nil {
return fmt.Errorf("postResult: %v", err)
}
- testResultString, pass, err := getResult(validatorId, resultsDir, false)
+ testResultString, pass, versionRecords, err := getResult(validatorId, resultsDir, false)
if err != nil {
return fmt.Errorf("postResult: couldn't parse results: %v", err)
}
@@ -772,9 +771,15 @@ func postResult(validatorId, version string) error {
return fmt.Errorf("postResult: couldn't create gist: %v", err)
}
+ if !pushToMaster && validatorId == "misc-checks" {
+ if err := postBreakingChangeLabel(g, versionRecords); err != nil {
+ return err
+ }
+ }
+
// Post parsed test results as a gist comment.
if _, err := g.AddGistComment(gistID, fmt.Sprintf("%s %s", commonci.Emoji(commonci.BoolStatusToString(pass)), validatorDesc), testResultString); err != nil {
- fmt.Errorf("postResult: could not add gist comment: %v", err)
+ return fmt.Errorf("postResult: could not add gist comment: %v", err)
}
prUpdate := &commonci.GithubPRUpdate{
diff --git a/post_results/main_test.go b/post_results/main_test.go
index 9849fce..7f7514d 100644
--- a/post_results/main_test.go
+++ b/post_results/main_test.go
@@ -171,7 +171,7 @@ func TestCheckSemverIncrease(t *testing.T) {
desc: "no change",
inOldVersion: "1.0.1",
inNewVersion: "1.0.1",
- wantErrSubstr: "file updated but PR version not updated",
+ wantErrSubstr: "file updated but test-version string not updated",
}, {
desc: "decrease",
inOldVersion: "1.0.1",
@@ -191,10 +191,74 @@ func TestCheckSemverIncrease(t *testing.T) {
for _, tt := range tests {
t.Run(tt.desc, func(t *testing.T) {
- err := checkSemverIncrease(tt.inOldVersion, tt.inNewVersion)
+ oldver, newver, err := checkSemverIncrease(tt.inOldVersion, tt.inNewVersion, "test-version")
if diff := errdiff.Substring(err, tt.wantErrSubstr); diff != "" {
t.Fatalf("did not get expected error, %s", diff)
}
+ if err == nil {
+ if got, want := oldver.String(), tt.inOldVersion; got != want {
+ t.Fatalf("old version: got %s, want %s", got, want)
+ }
+ if got, want := newver.String(), tt.inNewVersion; got != want {
+ t.Fatalf("old version: got %s, want %s", got, want)
+ }
+ }
+ })
+ }
+}
+
+func TestVersionRecords(t *testing.T) {
+ tests := []struct {
+ desc string
+ inVersionRecords versionRecordSlice
+ wantHasBreaking bool
+ }{{
+ desc: "has breaking",
+ inVersionRecords: versionRecordSlice{{
+ File: "openconfig-interface-submodule.yang",
+ OldMajorVersion: 0,
+ NewMajorVersion: 1,
+ OldVersion: "0.5.0",
+ NewVersion: "1.0.0",
+ }, {
+ File: "openconfig-interface.yang",
+ OldMajorVersion: 1,
+ NewMajorVersion: 2,
+ OldVersion: "1.1.3",
+ NewVersion: "2.0.0",
+ }},
+ wantHasBreaking: true,
+ }, {
+ desc: "non-breaking",
+ inVersionRecords: versionRecordSlice{{
+ File: "openconfig-interface-submodule.yang",
+ OldMajorVersion: 0,
+ NewMajorVersion: 1,
+ OldVersion: "0.5.0",
+ NewVersion: "1.0.0",
+ }},
+ wantHasBreaking: false,
+ }, {
+ desc: "non-breaking",
+ inVersionRecords: versionRecordSlice{{
+ File: "openconfig-interface-submodule.yang",
+ OldMajorVersion: 1,
+ NewMajorVersion: 1,
+ OldVersion: "1.5.0",
+ NewVersion: "1.6.0",
+ }},
+ wantHasBreaking: false,
+ }, {
+ desc: "empty",
+ inVersionRecords: nil,
+ wantHasBreaking: false,
+ }}
+
+ for _, tt := range tests {
+ t.Run(tt.desc, func(t *testing.T) {
+ if gotHasBreaking, want := tt.inVersionRecords.hasBreaking(), tt.wantHasBreaking; gotHasBreaking != want {
+ t.Errorf("gotHasBreaking %v, want %v", gotHasBreaking, want)
+ }
})
}
}
@@ -203,14 +267,15 @@ func TestGetResult(t *testing.T) {
modelRoot = "/workspace/release/yang"
tests := []struct {
- name string
- inValidatorResultDir string
- inValidatorId string
- wantPass bool
- wantOut string
- wantCondensedOut string
- wantCondensedOutSame bool
- wantErrSubstr string
+ name string
+ inValidatorResultDir string
+ inValidatorId string
+ wantPass bool
+ wantOut string
+ wantCondensedOut string
+ wantCondensedOutSame bool
+ wantVersionRecordSlice versionRecordSlice
+ wantErrSubstr string
}{{
name: "basic pyang pass",
inValidatorResultDir: "testdata/oc-pyang",
@@ -220,10 +285,12 @@ func TestGetResult(t *testing.T) {
✅ acl
✅ openconfig-acl
-
- 💲 bash command
+💲 bash command
foo command
-
+$OC_WORKSPACE/foo/bar
+$GOPATH/src/github.com/openconfig/oc-pyang/openconfig_pyang/plugins
+$GOPATH/src/github.com/robshakir/pyangbind/pyangbind/plugin
+
Passed.
@@ -263,10 +330,12 @@ Test failed with no stderr output.
✅ acl
✅ openconfig-acl
-
- 💲 bash command
+💲 bash command
foo command
-
+$OC_WORKSPACE/foo/bar
+$GOPATH/src/github.com/openconfig/oc-pyang/openconfig_pyang/plugins
+$GOPATH/src/github.com/robshakir/pyangbind/pyangbind/plugin
+
Passed.
@@ -450,14 +519,48 @@ Failed.
inValidatorResultDir: "testdata/misc-checks-pass",
inValidatorId: "misc-checks",
wantPass: true,
+ wantVersionRecordSlice: versionRecordSlice{{
+ File: "openconfig-acl-submodule.yang",
+ OldMajorVersion: 1,
+ NewMajorVersion: 1,
+ OldVersion: "1.1.3",
+ NewVersion: "1.2.3",
+ }, {
+ File: "openconfig-acl.yang",
+ OldMajorVersion: 1,
+ NewMajorVersion: 1,
+ OldVersion: "1.2.2",
+ NewVersion: "1.2.3",
+ }, {
+ File: "openconfig-interface-submodule.yang",
+ OldMajorVersion: 0,
+ NewMajorVersion: 1,
+ OldVersion: "0.5.0",
+ NewVersion: "1.0.0",
+ }, {
+ File: "openconfig-interface.yang",
+ OldMajorVersion: 1,
+ NewMajorVersion: 2,
+ OldVersion: "1.1.3",
+ NewVersion: "2.0.0",
+ }, {
+ File: "openconfig-packet-match.yang",
+ OldMajorVersion: 1,
+ NewMajorVersion: 1,
+ OldVersion: "1.1.2",
+ NewVersion: "1.2.0",
+ }},
wantOut: `
✅ openconfig-version update check
-6 file(s) correctly updated.
+9 file(s) correctly updated.
✅ .spec.yml build reachability check
-8 files reached by build rules.
+11 files reached by build rules.
+
+ ✅ submodule versions must match the belonging module's version
+7 module/submodule file groups have matching versions
`,
wantCondensedOutSame: true,
}, {
@@ -468,7 +571,7 @@ Failed.
wantOut: `
⛔ openconfig-version update check
changed-version-to-noversion.yang: openconfig-version was removed
- openconfig-acl.yang: file updated but PR version not updated: "1.2.2"
+ openconfig-acl.yang: file updated but openconfig-version string not updated: "1.2.2"
openconfig-mpls.yang: new semantic version not valid, old version: "2.3.4", new version: "2.2.5"
@@ -478,6 +581,10 @@ Failed.
changed-version-to-unreached.yang: file not used by any .spec.yml build.
unchanged-unreached.yang: file not used by any .spec.yml build.
+
+ ⛔ submodule versions must match the belonging module's version
+ module set openconfig-mpls is at 2.3.4 (openconfig-mpls-submodule.yang), non-matching files: openconfig-mpls-submodule2.yang (2.3.2), openconfig-mpls.yang (2.2.5)
+
`,
wantCondensedOutSame: true,
}}
@@ -485,7 +592,7 @@ Failed.
for _, tt := range tests {
for _, condensed := range []bool{false, true} {
t.Run(fmt.Sprintf(tt.name+"@condensed=%v", condensed), func(t *testing.T) {
- gotOut, gotPass, err := getResult(tt.inValidatorId, tt.inValidatorResultDir, condensed)
+ gotOut, gotPass, versionRecords, err := getResult(tt.inValidatorId, tt.inValidatorResultDir, condensed)
if err != nil {
if diff := errdiff.Substring(err, tt.wantErrSubstr); diff != "" {
t.Fatalf("did not get expected error, %s", diff)
@@ -495,6 +602,9 @@ Failed.
if gotPass != tt.wantPass {
t.Errorf("gotPass %v, want %v", gotPass, tt.wantPass)
}
+ if diff := cmp.Diff(tt.wantVersionRecordSlice, versionRecords); diff != "" {
+ t.Errorf("versionRecords (-want, +got):\n%s", diff)
+ }
wantOut := tt.wantOut
if condensed && !tt.wantCondensedOutSame {
wantOut = tt.wantCondensedOut
@@ -581,7 +691,7 @@ func TestWriteBadgeUploadCmdFile(t *testing.T) {
inValidatorUniqueStr: "pyang@latest",
inPass: true,
inResultsDir: "results-dir",
- wantFileContent: `REMOTE_PATH_PFX=gs://artifacts.disco-idea-817.appspot.com/compatibility-badges/openconfig-repo:
+ wantFileContent: `REMOTE_PATH_PFX=gs://openconfig/compatibility-badges/openconfig-repo:
RESULTSDIR=results-dir
upload-public-file() {
gsutil cp $RESULTSDIR/$1 "$REMOTE_PATH_PFX"$1
@@ -598,7 +708,7 @@ upload-public-file pyang@latest.html
inValidatorUniqueStr: "pyang",
inPass: false,
inResultsDir: "results-directory",
- wantFileContent: `REMOTE_PATH_PFX=gs://artifacts.disco-idea-817.appspot.com/compatibility-badges/openconfig-repo:
+ wantFileContent: `REMOTE_PATH_PFX=gs://openconfig/compatibility-badges/openconfig-repo:
RESULTSDIR=results-directory
upload-public-file() {
gsutil cp $RESULTSDIR/$1 "$REMOTE_PATH_PFX"$1
diff --git a/post_results/misc-checks.go b/post_results/misc-checks.go
new file mode 100644
index 0000000..9053aac
--- /dev/null
+++ b/post_results/misc-checks.go
@@ -0,0 +1,143 @@
+package main
+
+import (
+ "fmt"
+ "path/filepath"
+ "strings"
+
+ "github.com/Masterminds/semver/v3"
+ "github.com/openconfig/models-ci/commonci"
+)
+
+type versionRecord struct {
+ File string
+ OldMajorVersion uint64
+ NewMajorVersion uint64
+ OldVersion string
+ NewVersion string
+}
+
+type versionRecordSlice []versionRecord
+
+func (s versionRecordSlice) MajorVersionChanges() string {
+ if len(s) == 0 {
+ return fmt.Sprintf("No major YANG version changes in commit %s", commitSHA)
+ }
+
+ var b strings.Builder
+ b.WriteString(fmt.Sprintf("Major YANG version changes in commit %s:\n", commitSHA))
+ for _, change := range s {
+ if change.OldMajorVersion != change.NewMajorVersion {
+ b.WriteString(fmt.Sprintf("%s: `%s` -> `%s`\n", change.File, change.OldVersion, change.NewVersion))
+ }
+ }
+ return b.String()
+}
+
+func (s versionRecordSlice) hasBreaking() bool {
+ for _, change := range s {
+ if change.OldMajorVersion != 0 && change.OldMajorVersion != change.NewMajorVersion {
+ return true
+ }
+ }
+ return false
+}
+
+// processMiscChecksOutput takes the raw result output from the misc-checks
+// results directory and returns its formatted report and pass/fail status.
+//
+// It also returns a list of version changes for each file.
+func processMiscChecksOutput(resultsDir string) (string, bool, versionRecordSlice, error) {
+ fileProperties := map[string]map[string]string{}
+ changedFiles, err := readYangFilesList(filepath.Join(resultsDir, "changed-files.txt"))
+ if err != nil {
+ return "", false, nil, err
+ }
+ for _, file := range changedFiles {
+ if _, ok := fileProperties[file]; !ok {
+ fileProperties[file] = map[string]string{}
+ }
+ fileProperties[file]["changed"] = "true"
+ }
+ if err := readGoyangVersionsLog(filepath.Join(resultsDir, "pr-file-parse-log"), false, fileProperties); err != nil {
+ return "", false, nil, err
+ }
+ if err := readGoyangVersionsLog(filepath.Join(resultsDir, "master-file-parse-log"), true, fileProperties); err != nil {
+ return "", false, nil, err
+ }
+
+ var ocVersionViolations []string
+ ocVersionChangedCount := 0
+ var reachabilityViolations []string
+ filesReachedCount := 0
+ // Only look at the PR's files as they might be different from the master's files.
+ allNonEmptyPRFiles, err := readYangFilesList(filepath.Join(resultsDir, "all-non-empty-files.txt"))
+ if err != nil {
+ return "", false, nil, err
+ }
+ moduleFileGroups := map[string][]fileAndVersion{}
+ var versionRecords versionRecordSlice
+ for _, file := range allNonEmptyPRFiles {
+ properties, ok := fileProperties[file]
+
+ // Reachability check
+ if !ok || properties["reachable"] != "true" {
+ reachabilityViolations = append(reachabilityViolations, sprintLineHTML("%s: file not used by any .spec.yml build.", file))
+ // If the file was not reached, then its other
+ // parameters would not have been parsed by goyang, so
+ // simply skip the rest of the checks.
+ continue
+ }
+ filesReachedCount += 1
+
+ // openconfig-version update check
+ ocVersion, hasVersion := properties["openconfig-version"]
+ masterOcVersion, hadVersion := properties["master-openconfig-version"]
+ switch {
+ case properties["changed"] != "true":
+ // We assume the versioning is correct without change.
+ case hadVersion && hasVersion:
+ oldver, newver, err := checkSemverIncrease(masterOcVersion, ocVersion, "openconfig-version")
+ if err != nil {
+ ocVersionViolations = append(ocVersionViolations, sprintLineHTML(file+": "+err.Error()))
+ break
+ }
+ ocVersionChangedCount += 1
+ versionRecords = append(versionRecords, versionRecord{
+ File: file,
+ OldMajorVersion: oldver.Major(),
+ NewMajorVersion: newver.Major(),
+ OldVersion: masterOcVersion,
+ NewVersion: ocVersion,
+ })
+ case hadVersion && !hasVersion:
+ ocVersionViolations = append(ocVersionViolations, sprintLineHTML("%s: openconfig-version was removed", file))
+ default: // If didn't have version before, any new version is accepted.
+ ocVersionChangedCount += 1
+ }
+
+ if mod, ok := properties["belonging-module"]; hasVersion && ok {
+ // Error checking is already done by the version update check.
+ if v, err := semver.StrictNewVersion(ocVersion); err == nil {
+ moduleFileGroups[mod] = append(moduleFileGroups[mod], fileAndVersion{name: file, version: v})
+ }
+ }
+ }
+
+ // Compute HTML string and pass/fail status.
+ var out strings.Builder
+ var pass = true
+ appendViolationOut := func(desc string, violations []string, passString string) {
+ if len(violations) == 0 {
+ out.WriteString(sprintSummaryHTML(commonci.BoolStatusToString(true), desc, passString))
+ } else {
+ out.WriteString(sprintSummaryHTML(commonci.BoolStatusToString(false), desc, strings.Join(violations, "")))
+ pass = false
+ }
+ }
+ appendViolationOut("openconfig-version update check", ocVersionViolations, fmt.Sprintf("%d file(s) correctly updated.\n", ocVersionChangedCount))
+ appendViolationOut(".spec.yml build reachability check", reachabilityViolations, fmt.Sprintf("%d files reached by build rules.\n", filesReachedCount))
+ appendViolationOut("submodule versions must match the belonging module's version", versionGroupViolationsHTML(moduleFileGroups), fmt.Sprintf("%d module/submodule file groups have matching versions", len(moduleFileGroups)))
+
+ return out.String(), pass, versionRecords, nil
+}
diff --git a/post_results/testdata/misc-checks-fail/all-non-empty-files.txt b/post_results/testdata/misc-checks-fail/all-non-empty-files.txt
index bb42c59..4ab6b1f 100644
--- a/post_results/testdata/misc-checks-fail/all-non-empty-files.txt
+++ b/post_results/testdata/misc-checks-fail/all-non-empty-files.txt
@@ -1,4 +1,6 @@
release/models/mpls/openconfig-mpls.yang
+release/models/mpls/openconfig-mpls-submodule.yang
+release/models/mpls/openconfig-mpls-submodule2.yang
release/models/acl/deeper/openconfig-acl.yang
release/models/bgp/changed-unreached-to-unreached.yang
diff --git a/post_results/testdata/misc-checks-fail/master-file-parse-log b/post_results/testdata/misc-checks-fail/master-file-parse-log
index 32d6369..96ecb18 100644
--- a/post_results/testdata/misc-checks-fail/master-file-parse-log
+++ b/post_results/testdata/misc-checks-fail/master-file-parse-log
@@ -1,6 +1,8 @@
-openconfig-mpls.yang: openconfig-version:"2.3.4" latest-revision-version:"2.3.4"
-openconfig-acl.yang: openconfig-version:"1.2.2" latest-revision-version:"1.2.2"
+openconfig-mpls.yang: belonging-module:"openconfig-mpls" openconfig-version:"2.3.4" latest-revision-version:"2.3.4"
+openconfig-mpls-submodule.yang: belonging-module:"openconfig-mpls" openconfig-version:"2.3.4" latest-revision-version:"2.3.4"
+openconfig-mpls-submodule2.yang: belonging-module:"openconfig-mpls" openconfig-version:"2.3.2" latest-revision-version:"2.3.2"
+openconfig-acl.yang: belonging-module:"openconfig-acl" openconfig-version:"1.2.2" latest-revision-version:"1.2.2"
-changed-noversion-to-unreached.yang:
-changed-version-to-unreached.yang: openconfig-version:"1.0.0"
-changed-version-to-noversion.yang: openconfig-version:"1.0.0"
+changed-noversion-to-unreached.yang: belonging-module:"changed-noversion-to-unreached"
+changed-version-to-unreached.yang: belonging-module:"changed-version-to-unreached" openconfig-version:"1.0.0"
+changed-version-to-noversion.yang: belonging-module:"changed-version-to-noversion" openconfig-version:"1.0.0"
diff --git a/post_results/testdata/misc-checks-fail/pr-file-parse-log b/post_results/testdata/misc-checks-fail/pr-file-parse-log
index 5cdfe35..dbcfdae 100644
--- a/post_results/testdata/misc-checks-fail/pr-file-parse-log
+++ b/post_results/testdata/misc-checks-fail/pr-file-parse-log
@@ -1,3 +1,5 @@
-openconfig-mpls.yang: openconfig-version:"2.2.5" latest-revision-version:"2.2.5"
-openconfig-acl.yang: openconfig-version:"1.2.2" latest-revision-version:"1.2.2"
+openconfig-mpls.yang: belonging-module:"openconfig-mpls" openconfig-version:"2.2.5" latest-revision-version:"2.2.5"
+openconfig-mpls-submodule.yang: belonging-module:"openconfig-mpls" openconfig-version:"2.3.4" latest-revision-version:"2.3.4"
+openconfig-mpls-submodule2.yang: belonging-module:"openconfig-mpls" openconfig-version:"2.3.2" latest-revision-version:"2.3.2"
+openconfig-acl.yang: belonging-module:"openconfig-acl" openconfig-version:"1.2.2" latest-revision-version:"1.2.2"
changed-version-to-noversion.yang:
diff --git a/post_results/testdata/misc-checks-pass/all-non-empty-files.txt b/post_results/testdata/misc-checks-pass/all-non-empty-files.txt
index 0d6f569..3f7bd71 100644
--- a/post_results/testdata/misc-checks-pass/all-non-empty-files.txt
+++ b/post_results/testdata/misc-checks-pass/all-non-empty-files.txt
@@ -1,6 +1,9 @@
release/models/mpls/openconfig-mpls-static.yang
release/models/acl/openconfig-acl.yang
+release/models/acl/openconfig-acl-submodule.yang
release/models/acl/deeper/openconfig-packet-match.yang
+release/models/interfaces/openconfig-interface.yang
+release/models/interfaces/openconfig-interface-submodule.yang
release/models/bgp/changed-unreached-to-reached-noversion.yang
release/models/bgp/changed-unreached-to-version.yang
diff --git a/post_results/testdata/misc-checks-pass/changed-files.txt b/post_results/testdata/misc-checks-pass/changed-files.txt
index 8fd6c87..a99ec4b 100644
--- a/post_results/testdata/misc-checks-pass/changed-files.txt
+++ b/post_results/testdata/misc-checks-pass/changed-files.txt
@@ -1,8 +1,11 @@
release/models/acl/openconfig-acl.yang
+release/models/acl/openconfig-acl-submodule.yang
release/models/acl/deeper/openconfig-packet-match.yang
release/models/bgp/openconfig-deleted.yang
release/models/bgp/openconfig-deleted-2.yang
release/models/bgp/openconfig-deleted-3.yang
+release/models/interfaces/openconfig-interface.yang
+release/models/interfaces/openconfig-interface-submodule.yang
release/models/bgp/changed-unreached-to-reached-noversion.yang
release/models/bgp/changed-unreached-to-version.yang
diff --git a/post_results/testdata/misc-checks-pass/master-file-parse-log b/post_results/testdata/misc-checks-pass/master-file-parse-log
index 296d6ee..e610bbc 100644
--- a/post_results/testdata/misc-checks-pass/master-file-parse-log
+++ b/post_results/testdata/misc-checks-pass/master-file-parse-log
@@ -1,9 +1,12 @@
-openconfig-acl.yang: openconfig-version:"1.2.2"
-openconfig-packet-match.yang: latest-revision-version:"1.1.2" openconfig-version:"1.1.2"
-openconfig-mpls-static.yang: openconfig-version:"1.0.0"
-openconfig-deleted.yang: openconfig-version:"1.0.0"
-openconfig-deleted-2.yang:
+openconfig-acl.yang: openconfig-version:"1.2.2" belonging-module:"openconfig-acl"
+openconfig-acl-submodule.yang: openconfig-version:"1.1.3" belonging-module:"openconfig-acl"
+openconfig-packet-match.yang: latest-revision-version:"1.1.2" openconfig-version:"1.1.2" belonging-module:"openconfig-packet-match"
+openconfig-interface.yang: belonging-module:"openconfig-interface" openconfig-version:"1.1.3" latest-revision-version:"1.1.3"
+openconfig-interface-submodule.yang: belonging-module:"openconfig-interface-submodule" openconfig-version:"0.5.0" latest-revision-version:"0.5.0"
+openconfig-mpls-static.yang: openconfig-version:"1.0.0" belonging-module:"openconfig-mpls-static"
+openconfig-deleted.yang: openconfig-version:"1.0.0" belonging-module:"openconfig-deleted"
+openconfig-deleted-2.yang: belonging-module:"openconfig-deleted-2"
-changed-noversion-to-noversion.yang:
-changed-noversion-to-version.yang:
-unchanged-noversion.yang:
+changed-noversion-to-noversion.yang: belonging-module:"changed-noversion-to-noversion.yang"
+changed-noversion-to-version.yang: belonging-module:"changed-noversion-to-version"
+unchanged-noversion.yang: belonging-module:"unchanged-noversion"
diff --git a/post_results/testdata/misc-checks-pass/pr-file-parse-log b/post_results/testdata/misc-checks-pass/pr-file-parse-log
index 5530d87..dc41065 100644
--- a/post_results/testdata/misc-checks-pass/pr-file-parse-log
+++ b/post_results/testdata/misc-checks-pass/pr-file-parse-log
@@ -1,10 +1,13 @@
-openconfig-mpls-static.yang: openconfig-version:"1.0.0" latest-revision-version:"1.0.0"
-openconfig-acl.yang: openconfig-version:"1.2.3" latest-revision-version:"1.2.3"
-openconfig-packet-match.yang: latest-revision-version:"1.2.0" openconfig-version:"1.2.0"
-openconfig-mpls-static.yang: openconfig-version:"1.0.0" latest-revision-version:"1.0.0"
+openconfig-mpls-static.yang: belonging-module:"openconfig-mpls-static" openconfig-version:"1.0.0" latest-revision-version:"1.0.0"
+openconfig-acl.yang: belonging-module:"openconfig-acl" openconfig-version:"1.2.3" latest-revision-version:"1.2.3"
+openconfig-acl-submodule.yang: belonging-module:"openconfig-acl" openconfig-version:"1.2.3" latest-revision-version:"1.2.3"
+openconfig-packet-match.yang: belonging-module:"openconfig-packet-match" latest-revision-version:"1.2.0" openconfig-version:"1.2.0"
+openconfig-interface.yang: belonging-module:"openconfig-interface" openconfig-version:"2.0.0" latest-revision-version:"2.0.0"
+openconfig-interface-submodule.yang: belonging-module:"openconfig-interface-submodule" openconfig-version:"1.0.0" latest-revision-version:"1.0.0"
+openconfig-mpls-static.yang: belonging-module:"openconfig-mpls-static" openconfig-version:"1.0.0" latest-revision-version:"1.0.0"
-changed-unreached-to-reached-noversion.yang:
-changed-unreached-to-version.yang: openconfig-version:"1.0.0"
-changed-noversion-to-noversion.yang:
-changed-noversion-to-version.yang: openconfig-version:"1.0.0"
-unchanged-noversion.yang:
+changed-unreached-to-reached-noversion.yang: belonging-module:"changed-unreached-to-reached-noversion"
+changed-unreached-to-version.yang: belonging-module:"changed-unreached-to-version" openconfig-version:"1.0.0"
+changed-noversion-to-noversion.yang: belonging-module:"changed-noversion-to-noversion"
+changed-noversion-to-version.yang: belonging-module:"changed-noversion-to-version" openconfig-version:"1.0.0"
+unchanged-noversion.yang: belonging-module:"unchanged-noversion"
diff --git a/post_results/testdata/oc-pyang/acl==openconfig-acl==cmd b/post_results/testdata/oc-pyang/acl==openconfig-acl==cmd
index 23d5bfa..6c96d97 100644
--- a/post_results/testdata/oc-pyang/acl==openconfig-acl==cmd
+++ b/post_results/testdata/oc-pyang/acl==openconfig-acl==cmd
@@ -1 +1,4 @@
foo command
+/workspace/foo/bar
+$OCPYANG_PLUGIN_DIR
+$PYANGBIND_PLUGIN_DIR
diff --git a/util/parseutil.go b/util/parseutil.go
index 2ab7259..db296cf 100644
--- a/util/parseutil.go
+++ b/util/parseutil.go
@@ -1,3 +1,17 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
// Package util contain utility functions for doing YANG model validation.
package util
diff --git a/util/parseutil_test.go b/util/parseutil_test.go
index 6aab630..ca38d76 100644
--- a/util/parseutil_test.go
+++ b/util/parseutil_test.go
@@ -1,3 +1,17 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package util
import (
diff --git a/validators/compat_report.sh b/validators/compat_report.sh
index 4a65aaa..322627a 100755
--- a/validators/compat_report.sh
+++ b/validators/compat_report.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
ROOT_DIR=/workspace
USERCONFIG_DIR=$ROOT_DIR/user-config
diff --git a/validators/confd/test.sh b/validators/confd/test.sh
index 5e0e49d..60174f7 100755
--- a/validators/confd/test.sh
+++ b/validators/confd/test.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
ROOT_DIR=/workspace
ZIP_FILE=$ROOT_DIR/confd.zip
diff --git a/validators/goyang-ygot/test.sh b/validators/goyang-ygot/test.sh
index db6cef2..075c76a 100755
--- a/validators/goyang-ygot/test.sh
+++ b/validators/goyang-ygot/test.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
ROOT_DIR=/workspace
RESULTSDIR=$ROOT_DIR/results/goyang-ygot
@@ -9,8 +23,13 @@ if ! stat $RESULTSDIR; then
exit 0
fi
-GO111MODULE=on go list -m github.com/openconfig/ygot@latest > $RESULTSDIR/latest-version.txt
-if bash $RESULTSDIR/script.sh > $OUTFILE 2> $FAILFILE; then
+# module download logs go to stderr, so only fail if command failed.
+if ! go install github.com/openconfig/ygot/generator@latest &> "${OUTFILE}"; then
+ echo "failed: go install github.com/openconfig/ygot/generator@latest" > "${FAILFILE}"
+fi
+
+go list -m github.com/openconfig/ygot@latest > $RESULTSDIR/latest-version.txt
+if bash $RESULTSDIR/script.sh >> $OUTFILE 2>> $FAILFILE; then
# Delete fail file if it's empty and the script passed.
find $FAILFILE -size 0 -delete
fi
diff --git a/validators/misc-checks/ocversion/ocversion.go b/validators/misc-checks/ocversion/ocversion.go
index 6a28f5b..c1d6319 100644
--- a/validators/misc-checks/ocversion/ocversion.go
+++ b/validators/misc-checks/ocversion/ocversion.go
@@ -30,6 +30,15 @@ func init() {
flag.StringVar(&pathStr, "p", "", "comma separated list of directories to add to search path")
}
+// belongingModule returns the module name if m is a module and the belonging
+// module name if m is a submodule.
+func belongingModule(m *yang.Module) string {
+ if m.Kind() == "submodule" {
+ return m.BelongsTo.Name
+ }
+ return m.Name
+}
+
// ocVersionsList list all files with their openconfig-version value. If not
// present, it still lists the file.
// Any errors are reported to stderr.
@@ -43,6 +52,7 @@ func ocVersionsList(entries []*yang.Entry) string {
}
builder.WriteString(fmt.Sprintf("%s.yang:", m.Name))
+ builder.WriteString(fmt.Sprintf(" belonging-module:%q", belongingModule(m)))
for _, e := range m.Extensions {
keywordParts := strings.Split(e.Keyword, ":")
@@ -55,7 +65,7 @@ func ocVersionsList(entries []*yang.Entry) string {
extMod := yang.FindModuleByPrefix(m, pfx)
if extMod == nil {
builder.WriteString(fmt.Sprintf("unable to find module using prefix %q from referencing module %q\n", pfx, m.Name))
- } else if extMod.Name == "openconfig-extensions" {
+ } else if belongingModule(extMod) == "openconfig-extensions" {
builder.WriteString(fmt.Sprintf(" openconfig-version:%q", e.Argument))
}
}
@@ -67,6 +77,8 @@ func ocVersionsList(entries []*yang.Entry) string {
}
func buildModuleEntries(paths, files []string) ([]*yang.Entry, []error) {
+ ms := yang.NewModules()
+
var errs []error
for _, path := range paths {
expanded, err := yang.PathsWithModules(path)
@@ -74,11 +86,9 @@ func buildModuleEntries(paths, files []string) ([]*yang.Entry, []error) {
errs = append(errs, err)
continue
}
- yang.AddPath(expanded...)
+ ms.AddPath(expanded...)
}
- ms := yang.NewModules()
-
for _, name := range files {
if err := ms.Read(name); err != nil {
errs = append(errs, err)
diff --git a/validators/misc-checks/ocversion/ocversion_test.go b/validators/misc-checks/ocversion/ocversion_test.go
index 47eaab4..5ef4e14 100644
--- a/validators/misc-checks/ocversion/ocversion_test.go
+++ b/validators/misc-checks/ocversion/ocversion_test.go
@@ -29,18 +29,25 @@ func TestOcVersionsList(t *testing.T) {
want string
wantErr bool
}{{
- desc: "single extension",
- inPath: []string{"testdata"},
- inFiles: []string{"testdata/openconfig-single-extension.yang"},
- want: `openconfig-extensions.yang:
-openconfig-single-extension.yang: openconfig-version:"0.4.2"
+ desc: "module and submodule",
+ inPath: []string{"testdata"},
+ inFiles: []string{
+ "testdata/openconfig-extensions.yang",
+ "testdata/openconfig-extensions-submodule.yang",
+ "testdata/openconfig-single-extension.yang",
+ "testdata/openconfig-single-extension-submodule.yang",
+ },
+ want: `openconfig-extensions.yang: belonging-module:"openconfig-extensions"
+openconfig-extensions-submodule.yang: belonging-module:"openconfig-extensions" openconfig-version:"0.5.0"
+openconfig-single-extension.yang: belonging-module:"openconfig-single-extension" openconfig-version:"0.4.2"
+openconfig-single-extension-submodule.yang: belonging-module:"openconfig-single-extension" openconfig-version:"0.4.3"
`,
}, {
desc: "multiple extensions",
inPath: []string{"testdata"},
inFiles: []string{"testdata/openconfig-telemetry-types.yang"},
- want: `openconfig-extensions.yang:
-openconfig-telemetry-types.yang: openconfig-version:"0.4.2"
+ want: `openconfig-extensions.yang: belonging-module:"openconfig-extensions"
+openconfig-telemetry-types.yang: belonging-module:"openconfig-telemetry-types" openconfig-version:"0.4.2"
`,
}, {
desc: "invalid file",
@@ -51,9 +58,9 @@ openconfig-telemetry-types.yang: openconfig-version:"0.4.2"
desc: "other-extensions module used for openconfig-extension value",
inPath: []string{"testdata"},
inFiles: []string{"testdata/openconfig-use-other-extension.yang"},
- want: `openconfig-extensions.yang:
-openconfig-use-other-extension.yang:
-other-extensions.yang:
+ want: `openconfig-extensions.yang: belonging-module:"openconfig-extensions"
+openconfig-use-other-extension.yang: belonging-module:"openconfig-use-other-extension"
+other-extensions.yang: belonging-module:"other-extensions"
`,
}}
diff --git a/validators/misc-checks/ocversion/testdata/openconfig-extensions-submodule.yang b/validators/misc-checks/ocversion/testdata/openconfig-extensions-submodule.yang
new file mode 100644
index 0000000..f9577f1
--- /dev/null
+++ b/validators/misc-checks/ocversion/testdata/openconfig-extensions-submodule.yang
@@ -0,0 +1,9 @@
+submodule openconfig-extensions-submodule {
+
+ yang-version "1";
+
+ belongs-to openconfig-extensions { prefix oc-ext; }
+
+ oc-ext:openconfig-version "0.5.0";
+
+}
diff --git a/validators/misc-checks/ocversion/testdata/openconfig-single-extension-submodule.yang b/validators/misc-checks/ocversion/testdata/openconfig-single-extension-submodule.yang
new file mode 100644
index 0000000..33fa604
--- /dev/null
+++ b/validators/misc-checks/ocversion/testdata/openconfig-single-extension-submodule.yang
@@ -0,0 +1,11 @@
+submodule openconfig-single-extension-submodule {
+
+ yang-version "1";
+
+ belongs-to openconfig-single-extension { prefix oc-single-extension; }
+
+ import openconfig-extensions { prefix oc-ext; }
+
+ oc-ext:openconfig-version "0.4.3";
+
+}
diff --git a/validators/misc-checks/ocversion/testdata/openconfig-single-extension.yang b/validators/misc-checks/ocversion/testdata/openconfig-single-extension.yang
index 3f2d3ea..ad80840 100644
--- a/validators/misc-checks/ocversion/testdata/openconfig-single-extension.yang
+++ b/validators/misc-checks/ocversion/testdata/openconfig-single-extension.yang
@@ -3,12 +3,14 @@ module openconfig-single-extension {
yang-version "1";
// namespace
- namespace "http://openconfig.net/yang/telemetry-types";
+ namespace "http://openconfig.net/yang/single-extension";
- prefix "oc-telemetry-types";
+ prefix "oc-single-extension";
import openconfig-extensions { prefix oc-ext; }
+ include openconfig-single-extension-submodule;
+
oc-ext:openconfig-version "0.4.2";
}
diff --git a/validators/misc-checks/test.sh b/validators/misc-checks/test.sh
index bb4a4b9..c26835c 100755
--- a/validators/misc-checks/test.sh
+++ b/validators/misc-checks/test.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
ROOT_DIR=/workspace
RESULTSDIR=$ROOT_DIR/results/misc-checks
diff --git a/validators/oc-pyang/test.sh b/validators/oc-pyang/test.sh
index e76bbdc..f79bf79 100755
--- a/validators/oc-pyang/test.sh
+++ b/validators/oc-pyang/test.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
########################## SETUP #############################
ROOT_DIR=/workspace
@@ -19,10 +33,13 @@ setup() {
virtualenv $VENVDIR
source $VENVDIR/bin/activate
- git clone https://github.com/openconfig/oc-pyang $OCPYANG_REPO
+ git clone https://github.com/openconfig/oc-pyang $OCPYANG_REPO -b $_OC_PYANG_VERSION
+ cd $OCPYANG_REPO
+ echo -n "Running at github.com/openconfig/oc-pyang branch " >> $OUTFILE
+ git rev-parse --short HEAD >> $OUTFILE
+ cd ..
pip3 install --no-cache-dir -r $OCPYANG_DIR/requirements.txt
- pip3 install enum34
- pip3 install jinja2
+ pip3 install --no-cache-dir -r $OCPYANG_REPO/requirements.txt
pip3 install setuptools
pip3 install pyang
}
@@ -48,7 +65,7 @@ if [ $? -ne 0 ]; then
exit 0
fi
-if bash $RESULTSDIR/script.sh $VENVDIR/bin/pyang --plugindir $OCPYANG_PLUGIN_DIR > $OUTFILE 2> $FAILFILE; then
+if bash $RESULTSDIR/script.sh $VENVDIR/bin/pyang >> $OUTFILE 2> $FAILFILE; then
# Delete fail file if it's empty and the script passed.
find $FAILFILE -size 0 -delete
fi
diff --git a/validators/pyang/test.sh b/validators/pyang/test.sh
index 12cadf0..cbdc598 100755
--- a/validators/pyang/test.sh
+++ b/validators/pyang/test.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
########################## COMMON SETUP #############################
ROOT_DIR=/workspace
diff --git a/validators/pyangbind/test.sh b/validators/pyangbind/test.sh
index 4159e9b..0c46b07 100755
--- a/validators/pyangbind/test.sh
+++ b/validators/pyangbind/test.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
########################## SETUP #############################
ROOT_DIR=/workspace
@@ -20,7 +34,6 @@ setup() {
git clone https://github.com/robshakir/pyangbind $PYANGBIND_REPO
pip3 install --no-cache-dir -r $PYANGBIND_REPO/requirements.txt
- pip3 install pyangbind
pip3 install pyang
}
@@ -31,12 +44,13 @@ teardown() {
setup
########################## PYANGBIND #############################
-pip3 list | grep pyangbind > $RESULTSDIR/latest-version.txt
+echo -n "pyangbind@" > $RESULTSDIR/latest-version.txt
+cd "${PYANGBIND_REPO}" && git rev-parse --short HEAD >> $RESULTSDIR/latest-version.txt && cd -
find $RESULTSDIR/latest-version.txt -size 0 -delete
-export PYANGBIND_PLUGIN_DIR=`/usr/bin/env python3 -c \
- 'import pyangbind; import os; print ("{}/plugin".format(os.path.dirname(pyangbind.__file__)))'`
-if bash $RESULTSDIR/script.sh $VENVDIR/bin/pyang --plugindir $PYANGBIND_PLUGIN_DIR > $OUTFILE 2> $FAILFILE; then
+export PYTHONPATH="${PYTHONPATH}:${PYANGBIND_REPO}"
+export PYANGBIND_PLUGIN_DIR="${PYANGBIND_REPO}/pyangbind/plugin"
+if bash $RESULTSDIR/script.sh $VENVDIR/bin/pyang > $OUTFILE 2> $FAILFILE; then
# Delete fail file if it's empty and the script passed.
find $FAILFILE -size 0 -delete
fi
diff --git a/validators/regexp/test.sh b/validators/regexp/test.sh
index 94a8ef2..ffb5a6b 100755
--- a/validators/regexp/test.sh
+++ b/validators/regexp/test.sh
@@ -1,4 +1,18 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
########################## SETUP #############################
ROOT_DIR=/workspace
diff --git a/validators/yanglint/test.sh b/validators/yanglint/test.sh
index 8d701e7..5f228ac 100755
--- a/validators/yanglint/test.sh
+++ b/validators/yanglint/test.sh
@@ -1,7 +1,22 @@
#!/bin/bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
ROOT_DIR=/workspace
-DEB_FILE=$ROOT_DIR/libyang.deb
+DEB_FILE1=$ROOT_DIR/libyang.deb
+DEB_FILE2=$ROOT_DIR/yanglint.deb
RESULTSDIR=$ROOT_DIR/results/yanglint
OUTFILE=$RESULTSDIR/out
FAILFILE=$RESULTSDIR/fail
@@ -10,7 +25,8 @@ if ! stat $RESULTSDIR; then
exit 0
fi
-apt install $DEB_FILE
+apt install $DEB_FILE1
+apt install $DEB_FILE2
yanglint -v > $RESULTSDIR/latest-version.txt
if bash $RESULTSDIR/script.sh > $OUTFILE 2> $FAILFILE; then