From 8258fe20327129f9e93cf9d126cd8eaccd97ffd7 Mon Sep 17 00:00:00 2001
From: wenovus <wenbli@google.com>
Date: Tue, 21 Sep 2021 16:35:27 -0700
Subject: [PATCH] Minor fixes in functionality and style.

- Make some functions private.
- Minor UI fix.
- Crawler should check that a directory exists before crawling.
---
 frontend/static/query.html |  2 +-
 pkg/client/client.go       |  6 +++---
 pkg/db/db.go               | 34 +++++++++++++++++-----------------
 scripts/crawl/crawl.go     | 12 ++++++++----
 scripts/crawl/crawloc.sh   |  5 ++---
 5 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/frontend/static/query.html b/frontend/static/query.html
index 4aefae0..4afbad8 100644
--- a/frontend/static/query.html
+++ b/frontend/static/query.html
@@ -274,8 +274,8 @@
                     tableContent += `<th class="mdc-data-table__cell " scope="row ">` + modules[i].Version + `</th>`
                     if (queryType == "ModulesByOrgName" || queryType == "ModulesByKey") {
                         tableContent += `<th class="mdc-data-table__cell " scope="row "><a href="` + modules[i].URL + `">URL</a>` + `</th>`
+                        tableContent += `<th class="mdc-data-table__cell " scope="row ">` + modules[i].Summary + `</th>`
                     }
-                    tableContent += `<th class="mdc-data-table__cell " scope="row ">` + modules[i].Summary + `</th>`
                     tableContent += `<th class="mdc-data-table__cell " scope="row ">
                     <div class="mdc-touch-target-wrapper ">
                     <button class="mdc-button mdc-button--touch " onclick="download( '` + i + `') ">
diff --git a/pkg/client/client.go b/pkg/client/client.go
index d08392d..adc135a 100644
--- a/pkg/client/client.go
+++ b/pkg/client/client.go
@@ -46,7 +46,7 @@ func NewClient(filepath string) (*Client, error) {
 	token := ""
 	if filepath != "" {
 		var err error
-		token, err = ReadAuthToken(filepath)
+		token, err = readAuthToken(filepath)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create a new Client, read token failed: %v", err)
 		}
@@ -54,12 +54,12 @@ func NewClient(filepath string) (*Client, error) {
 	return &Client{token: token}, nil
 }
 
-// ReadAuthToken takes in *filepath* of token file, reads token and returns token string.
+// readAuthToken takes in *filepath* of token file, reads token and returns token string.
 // This token is used when server is deployed on Google Cloud Run and only avaiable to permitted users.
 // In this case, users need to include a header with identity token to get access to catalog server.
 // This token can be obtained via the command `glcoud auth print-identity-token`.
 // Reference: https://cloud.google.com/sdk/gcloud/reference/auth/print-identity-token.
-func ReadAuthToken(filepath string) (string, error) {
+func readAuthToken(filepath string) (string, error) {
 	token, err := ioutil.ReadFile(filepath)
 	if err != nil {
 		return "", fmt.Errorf("Read Auth token from file failed: %v", err)
diff --git a/pkg/db/db.go b/pkg/db/db.go
index 9e62310..3749279 100644
--- a/pkg/db/db.go
+++ b/pkg/db/db.go
@@ -16,7 +16,6 @@
 Package db contains functions related to database.
  * db.go includes conneting to db, query and insertion.
  * dbschema.go contains definitions of struct for db tables.
-   Currently it only contains Module struct.
 */
 package db
 
@@ -40,7 +39,8 @@ const (
 	selectModules = `select * from modules`
 	// We want to ensure that user has to provide all three inputs,
 	// instead of deleting too many modules by mistake with some fields missing.
-	deleteModule         = `delete from modules where orgName = $1 and name = $2 and version = $3`
+	deleteModule = `delete from modules where orgName = $1 and name = $2 and version = $3`
+
 	selectFeatureBundles = `select * from featureBundles`
 	// $4 and $5 should be assigned with the same value (the JSON data of feature-bundle).
 	insertFeatureBundle = `INSERT INTO featureBundles (orgName, name, version, data) VALUES($1, $2, $3, $4) on conflict (orgName, name, version) do update set data=$5`
@@ -161,10 +161,10 @@ func InsertModule(orgName string, name string, version string, data string) erro
 	return nil
 }
 
-// ReadModulesByRow scans from queried modules from rows one by one, rows are closed inside.
+// readModulesByRow scans from queried modules from rows one by one, rows are closed inside.
 // Return slice of db Module struct each field of which corresponds to one column in db.
 // Error is returned when scan rows failed.
-func ReadModulesByRow(rows *sql.Rows) ([]Module, error) {
+func readModulesByRow(rows *sql.Rows) ([]Module, error) {
 	var modules []Module
 	defer rows.Close()
 	for rows.Next() {
@@ -177,10 +177,10 @@ func ReadModulesByRow(rows *sql.Rows) ([]Module, error) {
 	return modules, nil
 }
 
-// FormatQueryStr is used to generate query statement string based on query parameters.
+// formatQueryStr is used to generate query statement string based on query parameters.
 // * parmNames is a list of names of all non-nil query parameters.
 // * baseQuery is query statement without any query parameters.
-func FormatQueryStr(parmNames []string, baseQuery string) string {
+func formatQueryStr(parmNames []string, baseQuery string) string {
 	queryStmt := baseQuery
 	for i := 0; i < len(parmNames); i++ {
 		if i == 0 {
@@ -207,7 +207,7 @@ func QueryModulesByOrgName(orgName *string) ([]Module, error) {
 	}
 
 	// Format query statement string based on non-nil query parameters
-	queryStmt := FormatQueryStr(parmNames, selectModules)
+	queryStmt := formatQueryStr(parmNames, selectModules)
 
 	rows, err := db.Query(queryStmt, parms...)
 	if err != nil {
@@ -216,7 +216,7 @@ func QueryModulesByOrgName(orgName *string) ([]Module, error) {
 
 	defer rows.Close()
 
-	return ReadModulesByRow(rows)
+	return readModulesByRow(rows)
 }
 
 // QueryModulesByKey queries modules by its key (name, version), it is possible that parameters are null.
@@ -238,7 +238,7 @@ func QueryModulesByKey(name *string, version *string) ([]Module, error) {
 	}
 
 	// Format query statement string based on non-nil query parameters
-	queryStmt := FormatQueryStr(parmNames, selectModules)
+	queryStmt := formatQueryStr(parmNames, selectModules)
 
 	rows, err := db.Query(queryStmt, parms...)
 	if err != nil {
@@ -247,7 +247,7 @@ func QueryModulesByKey(name *string, version *string) ([]Module, error) {
 
 	defer rows.Close()
 
-	return ReadModulesByRow(rows)
+	return readModulesByRow(rows)
 }
 
 // DeleteModule takes three string, orgName, name, version,
@@ -271,16 +271,16 @@ func DeleteModule(orgName string, name string, version string) error {
 	return nil
 }
 
-// ReadFeatureBundlesByRow scans from queried FeatureBundles from rows one by one, rows are closed inside.
+// readFeatureBundlesByRow scans from queried FeatureBundles from rows one by one, rows are closed inside.
 // Return slice of db FeatureBundle struct each field of which corresponds to one column in db.
 // Error is returned when scan rows failed.
-func ReadFeatureBundlesByRow(rows *sql.Rows) ([]FeatureBundle, error) {
+func readFeatureBundlesByRow(rows *sql.Rows) ([]FeatureBundle, error) {
 	var featureBundles []FeatureBundle
 	defer rows.Close()
 	for rows.Next() {
 		var featureBundle FeatureBundle
 		if err := rows.Scan(&featureBundle.OrgName, &featureBundle.Name, &featureBundle.Version, &featureBundle.Data); err != nil {
-			return nil, fmt.Errorf("ReadFeatureBundlesByRow: scan db rows failure, %v", err)
+			return nil, fmt.Errorf("readFeatureBundlesByRow: scan db rows failure, %v", err)
 		}
 		featureBundles = append(featureBundles, featureBundle)
 	}
@@ -301,14 +301,14 @@ func QueryFeatureBundlesByOrgName(orgName *string) ([]FeatureBundle, error) {
 	}
 
 	// Format query statement string based on non-nil query parameters
-	queryStmt := FormatQueryStr(parmNames, selectFeatureBundles)
+	queryStmt := formatQueryStr(parmNames, selectFeatureBundles)
 
 	rows, err := db.Query(queryStmt, parms...)
 	if err != nil {
 		return nil, fmt.Errorf("QueryFeatureBundlesByOrgName failed: %v", err)
 	}
 
-	return ReadFeatureBundlesByRow(rows)
+	return readFeatureBundlesByRow(rows)
 }
 
 // InsertFeatureBundle inserts FeatureBundle into database given values of four field of FeatureBundle schema.
@@ -361,12 +361,12 @@ func QueryFeatureBundlesByKey(name *string, version *string) ([]FeatureBundle, e
 	}
 
 	// Format query statement string based on non-nil query parameters
-	queryStmt := FormatQueryStr(parmNames, selectFeatureBundles)
+	queryStmt := formatQueryStr(parmNames, selectFeatureBundles)
 
 	rows, err := db.Query(queryStmt, parms...)
 	if err != nil {
 		return nil, fmt.Errorf("QueryFeatureBundlesByKey failed: %v", err)
 	}
 
-	return ReadFeatureBundlesByRow(rows)
+	return readFeatureBundlesByRow(rows)
 }
diff --git a/scripts/crawl/crawl.go b/scripts/crawl/crawl.go
index f757202..7d7635b 100644
--- a/scripts/crawl/crawl.go
+++ b/scripts/crawl/crawl.go
@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Program yang crawls all modules at certain commit of `https://github.com/openconfig/public`.
-// Usage: yang [--path DIR] [--url URL]
+// Program crawl crawls all modules at certain commit of `https://github.com/openconfig/public`.
+// Usage: crawl [--path DIR] [--url URL]
 //
 // DIR is a comma separated list of paths that are appended as the search directory.
 // If DIR appears as DIR/... then DIR and all direct and indirect subdirectories are checked.
@@ -146,6 +146,10 @@ func crawlModules(paths []string, url string) (map[string]*yang.Module, []string
 	// all modules in these directories and crawl them later.
 	if len(files) == 0 {
 		for _, path := range paths {
+			if _, err := os.Stat(path); err != nil {
+				log.Printf("WARNING: Cannot crawl in provided path: %v", err)
+				continue
+			}
 			names, err := traverseDir(path, url)
 			if err != nil {
 				log.Fatalf("traverse directory %s failed: %v", path, err)
@@ -280,8 +284,6 @@ func main() {
 		os.Exit(0)
 	}
 
-	mods, names := crawlModules(paths, url)
-
 	// Connect to DB
 	if err := db.ConnectDB(); err != nil {
 		log.Fatalf("Connect to db failed: %v", err)
@@ -289,6 +291,8 @@ func main() {
 	}
 	defer db.Close()
 
+	mods, names := crawlModules(paths, url)
+
 	// Convert all found modules into ygot go structure of Module and insert them into database.
 	for _, n := range names {
 		module := populateModule(mods[n], n)
diff --git a/scripts/crawl/crawloc.sh b/scripts/crawl/crawloc.sh
index f042f91..e0d3993 100644
--- a/scripts/crawl/crawloc.sh
+++ b/scripts/crawl/crawloc.sh
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-go build
 git clone https://github.com/openconfig/public.git
 # Copy ietf directory for backup usage.
 cp -r ./public/third_party/ietf .
@@ -37,7 +36,7 @@ do
     cd ..
     echo "./crawl -p $path -u https://raw.githubusercontent.com/openconfig/public/$commit/"
 
-    ./crawl -p $path -u https://raw.githubusercontent.com/openconfig/public/$commit/
+    go run crawl.go -p $path -u https://raw.githubusercontent.com/openconfig/public/$commit/
     cd public
     # Track how many commits the script has crawled.
     echo $times
@@ -50,4 +49,4 @@ done
 
 cd ..
 rm -rf public
-rm -rf ietf
\ No newline at end of file
+rm -rf ietf