Merge pull request #24 from yannh/fs-cache

Cache schemas downloaded over HTTP

Readme.md

@@ -13,7 +13,7 @@ It is inspired by, contains code from and is designed to stay close to
  * **high performance**: will validate & download manifests over multiple routines, caching
    downloaded files in memory
  * configurable list of **remote, or local schemas locations**, enabling validating Kubernetes
-   custom resources (CRDs)
+   custom resources (CRDs) and offline validation capabilities.
 
 ### A small overview of Kubernetes manifest validation
 
@@ -49,6 +49,10 @@ configuration errors.
 ```
 $ ./bin/kubeconform -h
 Usage: ./bin/kubeconform [OPTION]... [FILE OR FOLDER]...
+  -cache string
+        cache schemas downloaded via HTTP to this folder
+  -cpu-prof string
+        debug - log CPU profiling to file
   -exit-on-error
         immediately stop execution when the first error is encountered
   -h    show help information

acceptance.bats

@@ -202,3 +202,17 @@
   [ "$status" -eq 0 ]
   [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
 }
+
+@test "Pass when parsing a valid Kubernetes config YAML file and store cache" {
+  run mkdir cache
+  run bin/kubeconform -cache cache -summary fixtures/valid.yaml
+  [ "$status" -eq 0 ]
+  [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
+  [ "`ls cache/ | wc -l`" -eq 1 ]
+}
+
+@test "Fail when cache folder does not exist" {
+  run bin/kubeconform -cache cache_does_not_exist -summary fixtures/valid.yaml
+  [ "$status" -eq 1 ]
+  [ "$output" = "failed opening cache folder cache_does_not_exist: stat cache_does_not_exist: no such file or directory" ]
+}

cmd/kubeconform/main.go

@@ -82,6 +82,7 @@ func realMain() int {
 	}
 
 	v, err := validator.New(cfg.SchemaLocations, validator.Opts{
+		Cache:                cfg.Cache,
 		SkipTLS:              cfg.SkipTLS,
 		SkipKinds:            cfg.SkipKinds,
 		RejectKinds:          cfg.RejectKinds,

pkg/cache/cache.go

@@ -0,0 +1,6 @@
+package cache
+
+type Cache interface {
+	Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error)
+	Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error
+}

pkg/cache/inmemory.go

@@ -0,0 +1,49 @@
+package cache
+
+import (
+	"fmt"
+	"sync"
+)
+
+// SchemaCache is a cache for downloaded schemas, so each file is only retrieved once
+// It is different from pkg/registry/http_cache.go in that:
+//   - This cache caches the parsed Schemas
+type inMemory struct {
+	sync.RWMutex
+	schemas map[string]interface{}
+}
+
+// New creates a new cache for downloaded schemas
+func NewInMemoryCache() Cache {
+	return &inMemory{
+		schemas: map[string]interface{}{},
+	}
+}
+
+func key(resourceKind, resourceAPIVersion, k8sVersion string) string {
+	return fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion)
+}
+
+// Get retrieves the JSON schema given a resource signature
+func (c *inMemory) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) {
+	k := key(resourceKind, resourceAPIVersion, k8sVersion)
+	c.RLock()
+	defer c.RUnlock()
+	schema, ok := c.schemas[k]
+
+	if ok == false {
+		return nil, fmt.Errorf("schema not found in in-memory cache")
+	}
+
+	return schema, nil
+}
+
+// Set adds a JSON schema to the schema cache
+func (c *inMemory) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error {
+	k := key(resourceKind, resourceAPIVersion, k8sVersion)
+	c.Lock()
+	defer c.Unlock()
+	c.schemas[k] = schema
+
+	return nil
+}

pkg/cache/ondisk.go

@@ -0,0 +1,48 @@
+package cache
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sync"
+)
+
+type onDisk struct {
+	sync.RWMutex
+	folder string
+}
+
+// New creates a new cache for downloaded schemas
+func NewOnDiskCache(cache string) Cache {
+	return &onDisk{
+		folder: cache,
+	}
+}
+
+func cachePath(folder, resourceKind, resourceAPIVersion, k8sVersion string) string {
+	hash := md5.Sum([]byte(fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion)))
+	return path.Join(folder, hex.EncodeToString(hash[:]))
+}
+
+// Get retrieves the JSON schema given a resource signature
+func (c *onDisk) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) {
+	c.RLock()
+	defer c.RUnlock()
+
+	f, err := os.Open(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion))
+	if err != nil {
+		return nil, err
+	}
+
+	return ioutil.ReadAll(f)
+}
+
+// Set adds a JSON schema to the schema cache
+func (c *onDisk) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error {
+	c.Lock()
+	defer c.Unlock()
+	return ioutil.WriteFile(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion), schema.([]byte), 0644)
+}

pkg/config/config.go

@@ -9,6 +9,7 @@ import (
 )
 
 type Config struct {
+	Cache                  string
 	CPUProfileFile         string
 	ExitOnError            bool
 	Files                  []string
@@ -75,6 +76,7 @@ func FromFlags(progName string, args []string) (Config, string, error) {
 	flags.StringVar(&c.OutputFormat, "output", "text", "output format - json, tap, text")
 	flags.BoolVar(&c.Verbose, "verbose", false, "print results for all resources (ignored for tap output)")
 	flags.BoolVar(&c.SkipTLS, "insecure-skip-tls-verify", false, "disable verification of the server's SSL certificate. This will make your HTTPS connections insecure")
+	flags.StringVar(&c.Cache, "cache", "", "cache schemas downloaded via HTTP to this folder")
 	flags.StringVar(&c.CPUProfileFile, "cpu-prof", "", "debug - log CPU profiling to file")
 	flags.BoolVar(&c.Help, "h", false, "show help information")
 	flags.Usage = func() {

pkg/registry/http.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"time"
+
+	"github.com/yannh/kubeconform/pkg/cache"
 )
 
 type httpGetter interface {
@@ -16,10 +19,11 @@ type httpGetter interface {
 type SchemaRegistry struct {
 	c                  httpGetter
 	schemaPathTemplate string
+	cache              cache.Cache
 	strict             bool
 }
 
-func newHTTPRegistry(schemaPathTemplate string, strict bool, skipTLS bool) *SchemaRegistry {
+func newHTTPRegistry(schemaPathTemplate string, cacheFolder string, strict bool, skipTLS bool) (*SchemaRegistry, error) {
 	reghttp := &http.Transport{
 		MaxIdleConns:       100,
 		IdleConnTimeout:    3 * time.Second,
@@ -30,11 +34,25 @@ func newHTTPRegistry(schemaPathTemplate string, strict bool, skipTLS bool) *Sche
 		reghttp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}
 
+	var filecache cache.Cache = nil
+	if cacheFolder != "" {
+		fi, err := os.Stat(cacheFolder)
+		if err != nil {
+			return nil, fmt.Errorf("failed opening cache folder %s: %s", cacheFolder, err)
+		}
+		if !fi.IsDir() {
+			return nil, fmt.Errorf("cache folder %s is not a directory", err)
+		}
+
+		filecache = cache.NewOnDiskCache(cacheFolder)
+	}
+
 	return &SchemaRegistry{
 		c:                  &http.Client{Transport: reghttp},
 		schemaPathTemplate: schemaPathTemplate,
+		cache:              filecache,
 		strict:             strict,
-	}
+	}, nil
 }
 
 // DownloadSchema downloads the schema for a particular resource from an HTTP server
@@ -44,6 +62,12 @@ func (r SchemaRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVers
 		return nil, err
 	}
 
+	if r.cache != nil {
+		if b, err := r.cache.Get(resourceKind, resourceAPIVersion, k8sVersion); err == nil {
+			return b.([]byte), nil
+		}
+	}
+
 	resp, err := r.c.Get(url)
 	if err != nil {
 		return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err)
@@ -63,5 +87,11 @@ func (r SchemaRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVers
 		return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err)
 	}
 
+	if r.cache != nil {
+		if err := r.cache.Set(resourceKind, resourceAPIVersion, k8sVersion, body); err != nil {
+			return nil, fmt.Errorf("failed writing schema to cache: %s", err)
+		}
+	}
+
 	return body, nil
 }

pkg/registry/local.go

@@ -12,11 +12,11 @@ type LocalRegistry struct {
 }
 
 // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames
-func newLocalRegistry(pathTemplate string, strict bool) *LocalRegistry {
+func newLocalRegistry(pathTemplate string, strict bool) (*LocalRegistry, error) {
 	return &LocalRegistry{
 		pathTemplate,
 		strict,
-	}
+	}, nil
 }
 
 // DownloadSchema retrieves the schema from a file for the resource

pkg/registry/registry.go

@@ -79,7 +79,7 @@ func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict
 	return buf.String(), nil
 }
 
-func New(schemaLocation string, strict bool, skipTLS bool) (Registry, error) {
+func New(schemaLocation string, cache string, strict bool, skipTLS bool) (Registry, error) {
 	if !strings.HasSuffix(schemaLocation, "json") { // If we dont specify a full templated path, we assume the paths of kubernetesjsonschema.dev
 		schemaLocation += "/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json"
 	}
@@ -90,8 +90,8 @@ func New(schemaLocation string, strict bool, skipTLS bool) (Registry, error) {
 	}
 
 	if strings.HasPrefix(schemaLocation, "http") {
-		return newHTTPRegistry(schemaLocation, strict, skipTLS), nil
+		return newHTTPRegistry(schemaLocation, cache, strict, skipTLS)
 	}
 
-	return newLocalRegistry(schemaLocation, strict), nil
+	return newLocalRegistry(schemaLocation, strict)
 }

pkg/validator/validator.go

@@ -42,6 +42,7 @@ type Validator interface {
 
 // Opts contains a set of options for the validator.
 type Opts struct {
+	Cache                string              // Cache schemas downloaded via HTTP to this folder
 	SkipTLS              bool                // skip TLS validation when downloading from an HTTP Schema Registry
 	SkipKinds            map[string]struct{} // List of resource Kinds to ignore
 	RejectKinds          map[string]struct{} // List of resource Kinds to reject
@@ -59,7 +60,7 @@ func New(schemaLocations []string, opts Opts) (Validator, error) {
 
 	registries := []registry.Registry{}
 	for _, schemaLocation := range schemaLocations {
-		reg, err := registry.New(schemaLocation, opts.Strict, opts.SkipTLS)
+		reg, err := registry.New(schemaLocation, opts.Cache, opts.Strict, opts.SkipTLS)
 		if err != nil {
 			return nil, err
 		}
@@ -80,14 +81,14 @@ func New(schemaLocations []string, opts Opts) (Validator, error) {
 	return &v{
 		opts:           opts,
 		schemaDownload: downloadSchema,
-		schemaCache:    cache.New(),
+		schemaCache:    cache.NewInMemoryCache(),
 		regs:           registries,
 	}, nil
 }
 
 type v struct {
 	opts           Opts
-	schemaCache    *cache.SchemaCache
+	schemaCache    cache.Cache
 	schemaDownload func(registries []registry.Registry, kind, version, k8sVersion string) (*gojsonschema.Schema, error)
 	regs           []registry.Registry
 }
@@ -133,11 +134,13 @@ func (val *v) ValidateResource(res resource.Resource) Result {
 
 	cached := false
 	var schema *gojsonschema.Schema
-	cacheKey := ""
 
 	if val.schemaCache != nil {
-		cacheKey = cache.Key(sig.Kind, sig.Version, val.opts.KubernetesVersion)
-		schema, cached = val.schemaCache.Get(cacheKey)
+		s, err := val.schemaCache.Get(sig.Kind, sig.Version, val.opts.KubernetesVersion)
+		if err == nil {
+			cached = true
+			schema = s.(*gojsonschema.Schema)
+		}
 	}
 
 	if !cached {
@@ -146,7 +149,7 @@ func (val *v) ValidateResource(res resource.Resource) Result {
 		}
 
 		if val.schemaCache != nil {
-			val.schemaCache.Set(cacheKey, schema)
+			val.schemaCache.Set(sig.Kind, sig.Version, val.opts.KubernetesVersion, schema)
 		}
 	}