Merge branch 'master' into improve-error-msg

openconfig · Aug 4, 2023 · aa1ed2f · aa1ed2f
2 parents 20bbd0b + 41e0b30
commit aa1ed2f
Show file tree

Hide file tree

Showing 41 changed files with 848 additions and 209 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: Go
 
 on:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,29 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement (CLA). You (or your employer) retain the copyright to your
+contribution; this simply gives us permission to use and redistribute your
+contributions as part of the project. Head over to
+<https://cla.developers.google.com/> to see your current agreements on file or
+to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code Reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google/conduct/).
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,18 @@
-FROM gcr.io/cloud-builders/go:debian
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang
 SHELL ["/bin/bash", "-c"]
 
 #RUN git clone https://github.com/mbj4668/pyang.git /workspace/results/pyang@head/pyang

diff --git a/LICENSE b/LICENSE
@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -178,15 +179,15 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
       file or class name and description of purpose be included on the
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/README.md b/README.md
@@ -171,7 +171,7 @@ regexp            | Files moved into GOPATH from its folder during CI build
 pyang & pyangbind | pip
 oc-pyang          | git clone
 goyang/ygot       | go get
-yanglint          | Debian package periodically uploaded to cloud storage
+yanglint          | Debian packages (libyang2 and libyang2-tools) periodically uploaded to cloud storage. These are renamed libyang.deb and yanglint.deb respectively in the GCS bucket.
 
 ## Setting Up GCB
 

diff --git a/cmd_gen/main.go b/cmd_gen/main.go
@@ -122,7 +122,7 @@ var (
 	// GCB script, which together create the running environment for the
 	// generated validator script.
 	scriptTemplates = map[string]*scriptSpec{
-		"pyang": &scriptSpec{
+		"pyang": {
 			headerTemplate: mustTemplate("pyang-header", `#!/bin/bash
 workdir={{ .ResultsDir }}
 mkdir -p "$workdir"
@@ -138,7 +138,7 @@ script_options=(
 function run-dir() {
   declare prefix="$workdir"/"$1"=="$2"==
   shift 2
-  echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+  echo pyang "${options[@]}" "$@" > ${prefix}cmd
   if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
     mv ${prefix}pass ${prefix}fail
   fi
@@ -147,25 +147,27 @@ function run-dir() {
 			perModelTemplate: mustTemplate("pyang", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
 `),
 		},
-		"oc-pyang": &scriptSpec{
+		"oc-pyang": {
 			headerTemplate: mustTemplate("oc-pyang-header", `#!/bin/bash
 workdir={{ .ResultsDir }}
 mkdir -p "$workdir"
 `+"{{`"+util.PYANG_MSG_TEMPLATE_STRING+"`}}"+`
 cmd="$@"
 options=(
-  -p {{ .ModelRoot }}
-  -p {{ .RepoRoot }}/third_party/ietf
   --openconfig
   --ignore-error=OC_RELATIVE_PATH
+  -p {{ .ModelRoot }}
+  -p {{ .RepoRoot }}/third_party/ietf
 )
 script_options=(
   --msg-template "$PYANG_MSG_TEMPLATE"
 )
 function run-dir() {
   declare prefix="$workdir"/"$1"=="$2"==
+  local cmd_display_options=( --plugindir '$OCPYANG_PLUGIN_DIR' "${options[@]}" )
+  local options=( --plugindir "$OCPYANG_PLUGIN_DIR" "${options[@]}" )
   shift 2
-  echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+  echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
   if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
     mv ${prefix}pass ${prefix}fail
   fi
@@ -174,34 +176,41 @@ function run-dir() {
 			perModelTemplate: mustTemplate("oc-pyang", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
 `),
 		},
-		"pyangbind": &scriptSpec{
+		"pyangbind": {
 			headerTemplate: mustTemplate("pyangbind-header", `#!/bin/bash
 workdir={{ .ResultsDir }}
 mkdir -p "$workdir"
 `+"{{`"+util.PYANG_MSG_TEMPLATE_STRING+"`}}"+`
 cmd="$@"
 options=(
+  -f pybind
   -p {{ .ModelRoot }}
   -p {{ .RepoRoot }}/third_party/ietf
-  -f pybind
 )
 script_options=(
   --msg-template "$PYANG_MSG_TEMPLATE"
 )
 function run-dir() {
   declare prefix="$workdir"/"$1"=="$2"==
-  local options=( -o "$1"."$2".binding.py "${options[@]}" )
+  local output_file="$1"."$2".binding.py
+  local cmd_display_options=( --plugindir '$PYANGBIND_PLUGIN_DIR' -o "${output_file}" "${options[@]}" )
+  local options=( --plugindir "$PYANGBIND_PLUGIN_DIR" -o "${output_file}" "${options[@]}" )
   shift 2
-  echo $cmd "${options[@]}" "$@" > ${prefix}cmd
-  if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
+  echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
+  status=0
+  $cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
+  if [[ $status -eq "0" ]]; then
+    python "${output_file}" &>> ${prefix}pass || status=1
+  fi
+  if [[ $status -eq "1" ]]; then
     mv ${prefix}pass ${prefix}fail
   fi
 }
 `),
 			perModelTemplate: mustTemplate("pyangbind", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
 `),
 		},
-		"goyang-ygot": &scriptSpec{
+		"goyang-ygot": {
 			headerTemplate: mustTemplate("goyang-ygot-header", `#!/bin/bash
 workdir={{ .ResultsDir }}
 mkdir -p "$workdir"
@@ -226,8 +235,9 @@ function run-dir() {
   status=0
   $cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
   cd "$outdir"
-  go get &>> ${prefix}pass || status=1
   if [[ $status -eq "0" ]]; then
+    go mod init &>> ${prefix}pass || status=1
+    go mod tidy &>> ${prefix}pass || status=1
     go build &>> ${prefix}pass || status=1
   fi
   if [[ $status -eq "1" ]]; then
@@ -238,7 +248,7 @@ function run-dir() {
 			perModelTemplate: mustTemplate("goyang-ygot", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
 `),
 		},
-		"yanglint": &scriptSpec{
+		"yanglint": {
 			headerTemplate: mustTemplate("yanglint-header", `#!/bin/bash
 workdir={{ .ResultsDir }}
 mkdir -p "$workdir"
@@ -261,7 +271,7 @@ function run-dir() {
 			perModelTemplate: mustTemplate("yanglint", `run-dir "{{ .ModelDirName }}" "{{ .ModelName }}" {{- range $i, $buildFile := .BuildFiles }} {{ $buildFile }} {{- end }} {{- if .Parallel }} & {{- end }}
 `),
 		},
-		"confd": &scriptSpec{
+		"confd": {
 			headerTemplate: mustTemplate("confd-header", `#!/bin/bash
 workdir={{ .ResultsDir }}
 mkdir -p "$workdir"
@@ -275,7 +285,7 @@ if [[ $status -eq "1" ]]; then
 fi
 `),
 		},
-		"misc-checks": &scriptSpec{
+		"misc-checks": {
 			headerTemplate: mustTemplate("misc-checks-header", `#!/bin/bash
 workdir={{ .ResultsDir }}
 mkdir -p "$workdir"
@@ -341,6 +351,7 @@ type labelPoster interface {
 //     will be run only on a single model as specified in the .spec.yml file.
 //  2. Thus, a validation command and result is provided for each model.
 //  3. A file indicating pass/fail is output for each model into the given result directory.
+//
 // Files names follow the "modelDir==model==status" format with no file extensions.
 // The local flag indicates to run this as a helper to generate the script,
 // rather than running it within GCB.
@@ -461,7 +472,7 @@ func main() {
 	// If it's a push on master, just upload badge for normal validators as the only action.
 	if prNumber == 0 {
 		if branchName != "master" {
-			log.Fatalf("cmd_gen: There is no action to take for a non-master branch push, please re-examine your push triggers")
+			log.Fatalf("cmd_gen: pr-number not supplied as a flag to the build. Try re-running (by commenting \"/gcbrun\" on the GitHub PR) to see whether the $_PR_NUMBER substitution variable for Google Cloud Build gets passed into the build. If this branch is not associated with a PR, then it is inferred that this is a non-master branch push action, and thus there is no CI action that is expected, and in this case please re-examine your push triggers.")
 		}
 		pushToMaster = true
 	}

diff --git a/cmd_gen/main_test.go b/cmd_gen/main_test.go
@@ -66,7 +66,7 @@ script_options=(
 function run-dir() {
   declare prefix="$workdir"/"$1"=="$2"==
   shift 2
-  echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+  echo pyang "${options[@]}" "$@" > ${prefix}cmd
   if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
     mv ${prefix}pass ${prefix}fail
   fi
@@ -97,7 +97,7 @@ script_options=(
 function run-dir() {
   declare prefix="$workdir"/"$1"=="$2"==
   shift 2
-  echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+  echo pyang "${options[@]}" "$@" > ${prefix}cmd
   if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
     mv ${prefix}pass ${prefix}fail
   fi
@@ -116,18 +116,20 @@ mkdir -p "$workdir"
 PYANG_MSG_TEMPLATE='messages:{{path:"{file}" line:{line} code:"{code}" type:"{type}" level:{level} message:'"'{msg}'}}"
 cmd="$@"
 options=(
-  -p testdata
-  -p /workspace/third_party/ietf
   --openconfig
   --ignore-error=OC_RELATIVE_PATH
+  -p testdata
+  -p /workspace/third_party/ietf
 )
 script_options=(
   --msg-template "$PYANG_MSG_TEMPLATE"
 )
 function run-dir() {
   declare prefix="$workdir"/"$1"=="$2"==
+  local cmd_display_options=( --plugindir '$OCPYANG_PLUGIN_DIR' "${options[@]}" )
+  local options=( --plugindir "$OCPYANG_PLUGIN_DIR" "${options[@]}" )
   shift 2
-  echo $cmd "${options[@]}" "$@" > ${prefix}cmd
+  echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
   if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
     mv ${prefix}pass ${prefix}fail
   fi
@@ -147,19 +149,26 @@ mkdir -p "$workdir"
 PYANG_MSG_TEMPLATE='messages:{{path:"{file}" line:{line} code:"{code}" type:"{type}" level:{level} message:'"'{msg}'}}"
 cmd="$@"
 options=(
+  -f pybind
   -p testdata
   -p /workspace/third_party/ietf
-  -f pybind
 )
 script_options=(
   --msg-template "$PYANG_MSG_TEMPLATE"
 )
 function run-dir() {
   declare prefix="$workdir"/"$1"=="$2"==
-  local options=( -o "$1"."$2".binding.py "${options[@]}" )
+  local output_file="$1"."$2".binding.py
+  local cmd_display_options=( --plugindir '$PYANGBIND_PLUGIN_DIR' -o "${output_file}" "${options[@]}" )
+  local options=( --plugindir "$PYANGBIND_PLUGIN_DIR" -o "${output_file}" "${options[@]}" )
   shift 2
-  echo $cmd "${options[@]}" "$@" > ${prefix}cmd
-  if ! $($cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass); then
+  echo pyang "${cmd_display_options[@]}" "$@" > ${prefix}cmd
+  status=0
+  $cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
+  if [[ $status -eq "0" ]]; then
+    python "${output_file}" &>> ${prefix}pass || status=1
+  fi
+  if [[ $status -eq "1" ]]; then
     mv ${prefix}pass ${prefix}fail
   fi
 }
@@ -196,8 +205,9 @@ function run-dir() {
   status=0
   $cmd "${options[@]}" "${script_options[@]}" "$@" &> ${prefix}pass || status=1
   cd "$outdir"
-  go get &>> ${prefix}pass || status=1
   if [[ $status -eq "0" ]]; then
+    go mod init &>> ${prefix}pass || status=1
+    go mod tidy &>> ${prefix}pass || status=1
     go build &>> ${prefix}pass || status=1
   fi
   if [[ $status -eq "1" ]]; then

diff --git a/cmd_gen/testdata/acl/.spec.yml b/cmd_gen/testdata/acl/.spec.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 - name: openconfig-acl
   docs:
     - yang/acl/openconfig-packet-match-types.yang

diff --git a/cmd_gen/testdata/optical-transport/.spec.yml b/cmd_gen/testdata/optical-transport/.spec.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 - name: openconfig-terminal-device
   docs:
     - yang/optical-transport/openconfig-transport-types.yang