Refactor logging and enhance RDF metadata generation

Replace standard log package with logrus for enhanced logging capabilities across the project. Update RDF metadata generation to include dynamic timestamps and bucket-derived names, improving the accuracy and relevance of generated data descriptions.
gleanerio · Nov 15, 2024 · c897b6f · c897b6f
1 parent 22099ff
commit c897b6f
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 11 deletions.
diff --git a/docs/README.md b/docs/README.md
@@ -177,4 +177,12 @@ nabu object --cfgPath directory --cfgName name  objectId
 eg use generated
 ```
 nabu object --cfgPath ../gleaner/configs --cfgName local milled/opentopography/ffa0df033bb3a8fc9f600c80df3501fe1a2dbe93.rdf
+```
+
+### Using URL based configuration
+
+Nabu can also read the configuration file from over the network
+
+```
+go run ../../cmd/nabu/main.go release --cfgURL https://provisium.io/data/nabuconfig.yaml --prefix summoned/dataverse --endpoint localoxi
 ```
diff --git a/internal/graph/toFromRDF.go b/internal/graph/toFromRDF.go
@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"log"
+	log "github.com/sirupsen/logrus"
 	"strings"
 
 	"github.com/knakk/rdf"

diff --git a/internal/objects/pipecopy.go b/internal/objects/pipecopy.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/spf13/viper"
 
@@ -17,6 +18,11 @@ import (
 	"github.com/minio/minio-go/v7"
 )
 
+func getLastElement(s string) string {
+	parts := strings.Split(s, "/")
+	return parts[len(parts)-1]
+}
+
 // PipeCopy writes a new object based on an prefix, this function assumes the objects are valid when concatenated
 // v1:  viper config object
 // mc:  minio client pointer
@@ -118,19 +124,24 @@ func PipeCopy(v1 *viper.Viper, mc *minio.Client, name, bucket, prefix, destprefi
 		if lastProcessed {
 
 			data := `_:b0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://schema.org/DataCatalog> .
-_:b0 <https://schema.org/dateCreated> "2024-09-20" .
-_:b0 <https://schema.org/description> "This is an example data catalog containing various datasets from this organization" .
+_:b0 <https://schema.org/dateCreated> "` + time.Now().Format("2006-01-02 15:04:05") + `" . 
+_:b0 <https://schema.org/description> "GleanerIO Nabu generated catalog" .
 _:b0 <https://schema.org/provider> _:b1 .
 _:b0 <https://schema.org/publisher> _:b2 .
 _:b1 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://schema.org/Organization> .
-_:b1 <https://schema.org/name> "Provider XYZ" .
+_:b1 <https://schema.org/name> "` + getLastElement(prefix) + `" .
 _:b2 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://schema.org/Organization> .
-_:b2 <https://schema.org/name> "DeCoder" .
-`
+_:b2 <https://schema.org/name> "` + bucket + `" .`
+
 			for _, item := range idList {
 				data += `_:b0 <https://schema.org/dataset> <` + item + `> .` + "\n"
 			}
 
+			// TODO MakeURN with _:b0   Q's Will this work with a blank node? do after Skolemization?
+			// namedgraph, err := graph.MakeURN(v1, "resource IRI")
+			// sdataWithContext, err := graph.NtToNq(sdata, namedgraph)
+
+			// TODO:  Skolemize with sdataWithContext
 			sdata, err := graph.Skolemization(data, "release graph prov for ORG")
 			if err != nil {
 				log.Println(err)

diff --git a/pkg/config/nabuConfig.go b/pkg/config/nabuConfig.go
@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 	"io"
 	"net/http"
@@ -41,7 +42,7 @@ func ReadNabuConfigURL(configURL string) (*viper.Viper, error) {
 		v.SetDefault(key, value)
 	}
 
-	fmt.Printf("Reading config from URL: %v\n", configURL)
+	log.Printf("Reading config from URL: %v\n", configURL)
 
 	resp, err := http.Get(configURL)
 	if err != nil {
@@ -83,9 +84,5 @@ func ReadNabuConfigURL(configURL string) (*viper.Viper, error) {
 		return v, err
 	}
 
-	fmt.Printf("Config read from URL: %v\n", v.AllSettings())
-
-	//err = v.ReadInConfig()
-
 	return v, err
 }