feat(chore): cache_keys regexp ordered (#331)

* feat(chore): cache_keys regexp ordered * feat: order properly the cache keys ordering * fix: unit tests * fix: caddy JSON parsing * fix: Caddy JSON marshaler, traefik v3.0+ support * fix: update caddy configuration.json * feat: bump version to the next release
darkweak · Mar 29, 2023 · 195397f · 195397f
1 parent 103602c
commit 195397f
Show file tree

Hide file tree

Showing 51 changed files with 4,809 additions and 4,479 deletions.
diff --git a/README.md b/README.md
@@ -925,7 +925,7 @@ experimental:
   plugins:
     souin:
       moduleName: github.com/darkweak/souin
-      version: v1.6.35
+      version: v1.6.36
 ```
 After that you can declare either the whole configuration at once in the middleware block or by service. See the examples below.
 ```yaml

diff --git a/configurationtypes/types.go b/configurationtypes/types.go
@@ -1,14 +1,106 @@
 package configurationtypes
 
 import (
+	"bytes"
 	"encoding/json"
+	"fmt"
 	"regexp"
+	"strconv"
+	"strings"
 	"time"
 
 	"go.uber.org/zap"
 	yaml "gopkg.in/yaml.v3"
 )
 
+type CacheKey map[RegValue]Key
+type CacheKeys []CacheKey
+
+func (c *CacheKeys) parseJSON(rootDecoder *json.Decoder) {
+	var token json.Token
+	var err error
+
+	_, _ = rootDecoder.Token()
+	_, _ = rootDecoder.Token()
+	_, _ = rootDecoder.Token()
+
+	for err == nil {
+		token, err = rootDecoder.Token()
+		key := Key{}
+		rg := fmt.Sprint(token)
+
+		value := fmt.Sprint(token)
+		if value == "}" || token == nil {
+			continue
+		}
+		for value != "}" && token != nil {
+			token, _ = rootDecoder.Token()
+			value = fmt.Sprint(token)
+			switch value {
+			case "disable_body":
+				val, _ := rootDecoder.Token()
+				key.DisableBody, _ = strconv.ParseBool(fmt.Sprint(val))
+			case "disable_host":
+				val, _ := rootDecoder.Token()
+				key.DisableHost, _ = strconv.ParseBool(fmt.Sprint(val))
+			case "disable_method":
+				val, _ := rootDecoder.Token()
+				key.DisableMethod, _ = strconv.ParseBool(fmt.Sprint(val))
+			case "disable_query":
+				val, _ := rootDecoder.Token()
+				key.DisableQuery, _ = strconv.ParseBool(fmt.Sprint(val))
+			case "hide":
+				val, _ := rootDecoder.Token()
+				key.Hide, _ = strconv.ParseBool(fmt.Sprint(val))
+			case "headers":
+				val, _ := rootDecoder.Token()
+				key.Headers = []string{}
+				for fmt.Sprint(val) != "]" {
+					val, _ = rootDecoder.Token()
+					header := fmt.Sprint(val)
+					if header != "]" {
+						key.Headers = append(key.Headers, header)
+					}
+				}
+			}
+		}
+		*c = append(*c, CacheKey{
+			{Regexp: regexp.MustCompile(rg)}: key,
+		})
+	}
+}
+
+func (c *CacheKeys) UnmarshalYAML(value *yaml.Node) error {
+	for i := 0; i < len(value.Content)/2; i++ {
+		var cacheKey CacheKey
+		err := value.Decode(&cacheKey)
+		if err != nil {
+			return err
+		}
+		*c = append(*c, cacheKey)
+	}
+
+	return nil
+}
+
+func (c *CacheKeys) UnmarshalJSON(value []byte) error {
+	c.parseJSON(json.NewDecoder(bytes.NewBuffer(value)))
+
+	return nil
+}
+
+func (c *CacheKeys) MarshalJSON() ([]byte, error) {
+	var strKeys []string
+	for _, cacheKey := range *c {
+		for rg, key := range cacheKey {
+			keyBytes, _ := json.Marshal(key)
+			strKeys = append(strKeys, fmt.Sprintf(`"%s": %v`, rg.Regexp.String(), string(keyBytes)))
+		}
+	}
+
+	return []byte(fmt.Sprintf(`{%s}`, strings.Join(strKeys, ","))), nil
+}
+
 // Duration is the super object to wrap the duration and be able to parse it from the configuration
 type Duration struct {
 	time.Duration
@@ -112,12 +204,12 @@ type CDN struct {
 }
 
 type Key struct {
-	DisableBody   bool     `json:"disable_body" yaml:"disable_body"`
-	DisableHost   bool     `json:"disable_host" yaml:"disable_host"`
-	DisableMethod bool     `json:"disable_method" yaml:"disable_method"`
-	DisableQuery  bool     `json:"disable_query" yaml:"disable_query"`
-	Hide          bool     `json:"hide" yaml:"hide"`
-	Headers       []string `json:"headers" yaml:"headers"`
+	DisableBody   bool     `json:"disable_body,omitempty" yaml:"disable_body,omitempty"`
+	DisableHost   bool     `json:"disable_host,omitempty" yaml:"disable_host,omitempty"`
+	DisableMethod bool     `json:"disable_method,omitempty" yaml:"disable_method,omitempty"`
+	DisableQuery  bool     `json:"disable_query,omitempty" yaml:"disable_query,omitempty"`
+	Hide          bool     `json:"hide,omitempty" yaml:"hide,omitempty"`
+	Headers       []string `json:"headers,omitempty" yaml:"headers,omitempty"`
 }
 
 // DefaultCache configuration
@@ -287,5 +379,5 @@ type AbstractConfigurationInterface interface {
 	SetLogger(*zap.Logger)
 	GetYkeys() map[string]SurrogateKeys
 	GetSurrogateKeys() map[string]SurrogateKeys
-	GetCacheKeys() map[RegValue]Key
+	GetCacheKeys() CacheKeys
 }
diff --git a/context/key.go b/context/key.go
@@ -21,7 +21,7 @@ type keyContext struct {
 	disable_query  bool
 	displayable    bool
 	headers        []string
-	overrides      map[*regexp.Regexp]keyContext
+	overrides      []map[*regexp.Regexp]keyContext
 }
 
 func (g *keyContext) SetupContext(c configurationtypes.AbstractConfigurationInterface) {
@@ -33,16 +33,18 @@ func (g *keyContext) SetupContext(c configurationtypes.AbstractConfigurationInte
 	g.displayable = !k.Hide
 	g.headers = k.Headers
 
-	g.overrides = make(map[*regexp.Regexp]keyContext)
-
-	for r, v := range c.GetCacheKeys() {
-		g.overrides[r.Regexp] = keyContext{
-			disable_body:   v.DisableBody,
-			disable_host:   v.DisableHost,
-			disable_method: v.DisableMethod,
-			disable_query:  v.DisableQuery,
-			displayable:    v.Hide,
-			headers:        v.Headers,
+	g.overrides = make([]map[*regexp.Regexp]keyContext, 0)
+
+	for _, cacheKey := range c.GetCacheKeys() {
+		for r, v := range cacheKey {
+			g.overrides = append(g.overrides, map[*regexp.Regexp]keyContext{r.Regexp: {
+				disable_body:   v.DisableBody,
+				disable_host:   v.DisableHost,
+				disable_method: v.DisableMethod,
+				disable_query:  v.DisableQuery,
+				displayable:    !v.Hide,
+				headers:        v.Headers,
+			}})
 		}
 	}
 }
@@ -83,30 +85,39 @@ func (g *keyContext) SetContext(req *http.Request) *http.Request {
 		headerValues += "-" + req.Header.Get(hn)
 	}
 
-	for k, v := range g.overrides {
-		if k.MatchString(req.RequestURI) {
-			displayable = v.displayable
-			host = ""
-			method = ""
-			if !v.disable_query && len(req.URL.RawQuery) > 0 {
-				query = "?" + req.URL.RawQuery
-			}
-			if !v.disable_body {
-				body = req.Context().Value(HashBody).(string)
-			}
-			if !v.disable_method {
-				method = req.Method + "-"
-			}
-			if !v.disable_host {
-				host = req.Host + "-"
-			}
-			if len(v.headers) > 0 {
-				headerValues = ""
-				for _, hn := range v.headers {
-					headers = v.headers
-					headerValues += "-" + req.Header.Get(hn)
+	hasOverride := false
+	for _, current := range g.overrides {
+		for k, v := range current {
+			if k.MatchString(req.RequestURI) {
+				displayable = v.displayable
+				host = ""
+				method = ""
+				query = ""
+				if !v.disable_query && len(req.URL.RawQuery) > 0 {
+					query = "?" + req.URL.RawQuery
+				}
+				if !v.disable_body {
+					body = req.Context().Value(HashBody).(string)
 				}
+				if !v.disable_method {
+					method = req.Method + "-"
+				}
+				if !v.disable_host {
+					host = req.Host + "-"
+				}
+				if len(v.headers) > 0 {
+					headerValues = ""
+					for _, hn := range v.headers {
+						headers = v.headers
+						headerValues += "-" + req.Header.Get(hn)
+					}
+				}
+				hasOverride = true
+				break
 			}
+		}
+
+		if hasOverride {
 			break
 		}
 	}

diff --git a/context/key_test.go b/context/key_test.go
@@ -25,13 +25,16 @@ func Test_KeyContext_SetupContext(t *testing.T) {
 		t.Errorf("The method must be disabled.")
 	}
 
-	m := make(map[configurationtypes.RegValue]configurationtypes.Key)
 	rg := configurationtypes.RegValue{
 		Regexp: regexp.MustCompile(".*"),
 	}
-	m[rg] = configurationtypes.Key{
-		DisableHost:   true,
-		DisableMethod: true,
+	m := configurationtypes.CacheKeys{
+		{
+			rg: configurationtypes.Key{
+				DisableHost:   true,
+				DisableMethod: true,
+			},
+		},
 	}
 	ctx.SetupContext(&configuration.Configuration{
 		DefaultCache: &configurationtypes.DefaultCache{
@@ -49,10 +52,10 @@ func Test_KeyContext_SetupContext(t *testing.T) {
 	if !ctx.disable_method {
 		t.Errorf("The method must be enabled.")
 	}
-	if !ctx.overrides[rg.Regexp].disable_host {
+	if !ctx.overrides[0][rg.Regexp].disable_host {
 		t.Errorf("The host must be enabled.")
 	}
-	if !ctx.overrides[rg.Regexp].disable_method {
+	if !ctx.overrides[0][rg.Regexp].disable_method {
 		t.Errorf("The method must be enabled.")
 	}
 }
@@ -65,30 +68,32 @@ func Test_KeyContext_SetContext(t *testing.T) {
 		t.Errorf("The Key context must be equal to GET-http-domain.com--with_the_hash, %s given.", req.Context().Value(Key).(string))
 	}
 
-	m := make(map[*regexp.Regexp]keyContext)
-	m[regexp.MustCompile("/matched")] = keyContext{
-		disable_host:   false,
-		disable_method: true,
+	m := map[*regexp.Regexp]keyContext{
+		regexp.MustCompile("/matched"): {
+			disable_host:   true,
+			disable_method: false,
+		},
 	}
 	ctx2 := keyContext{
 		disable_host:   true,
 		disable_method: true,
-		overrides:      m,
+		overrides:      []map[*regexp.Regexp]keyContext{m},
 	}
 	req2 := httptest.NewRequest(http.MethodGet, "http://domain.com/matched", nil)
 	req2 = ctx2.SetContext(req2.WithContext(context.WithValue(req2.Context(), HashBody, "")))
-	if req2.Context().Value(Key).(string) != "http-domain.com-/matched" {
-		t.Errorf("The Key context must be equal to http-domain.com-/matched, %s given.", req2.Context().Value(Key).(string))
+	if req2.Context().Value(Key).(string) != "GET-http-/matched" {
+		t.Errorf("The Key context must be equal to GET-http-/matched, %s given.", req2.Context().Value(Key).(string))
 	}
 
-	m = make(map[*regexp.Regexp]keyContext)
-	m[regexp.MustCompile("/matched")] = keyContext{
-		disable_host:   true,
-		disable_method: false,
+	m = map[*regexp.Regexp]keyContext{
+		regexp.MustCompile("/matched"): {
+			disable_host:   true,
+			disable_method: false,
+		},
 	}
 	ctx3 := keyContext{
 		disable_method: true,
-		overrides:      m,
+		overrides:      []map[*regexp.Regexp]keyContext{m},
 	}
 	req3 := httptest.NewRequest(http.MethodGet, "http://domain.com/matched", nil)
 	req3 = ctx3.SetContext(req3.WithContext(context.WithValue(req3.Context(), HashBody, "")))

diff --git a/pkg/middleware/configuration.go b/pkg/middleware/configuration.go
@@ -7,11 +7,11 @@ import (
 
 // BaseConfiguration holder
 type BaseConfiguration struct {
-	DefaultCache  *configurationtypes.DefaultCache                       `json:"default_cache" yaml:"default_cache"`
-	API           configurationtypes.API                                 `json:"api" yaml:"api"`
-	CacheKeys     map[configurationtypes.RegValue]configurationtypes.Key `yaml:"cache_keys"`
-	URLs          map[string]configurationtypes.URL                      `json:"urls" yaml:"urls"`
-	LogLevel      string                                                 `json:"log_level" yaml:"log_level"`
+	DefaultCache  *configurationtypes.DefaultCache  `json:"default_cache" yaml:"default_cache"`
+	API           configurationtypes.API            `json:"api" yaml:"api"`
+	CacheKeys     configurationtypes.CacheKeys      `json:"cache_keys" yaml:"cache_keys"`
+	URLs          map[string]configurationtypes.URL `json:"urls" yaml:"urls"`
+	LogLevel      string                            `json:"log_level" yaml:"log_level"`
 	Logger        *zap.Logger
 	Ykeys         map[string]configurationtypes.SurrogateKeys `json:"ykeys" yaml:"ykeys"`
 	SurrogateKeys map[string]configurationtypes.SurrogateKeys `json:"surrogate_keys" yaml:"surrogate_keys"`
@@ -58,7 +58,7 @@ func (c *BaseConfiguration) GetSurrogateKeys() map[string]configurationtypes.Sur
 }
 
 // GetCacheKeys get the cache keys rules to override
-func (c *BaseConfiguration) GetCacheKeys() map[configurationtypes.RegValue]configurationtypes.Key {
+func (c *BaseConfiguration) GetCacheKeys() configurationtypes.CacheKeys {
 	return c.CacheKeys
 }
 

diff --git a/plugins/beego/go.mod b/plugins/beego/go.mod
@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/beego/beego/v2 v2.0.7
-	github.com/darkweak/souin v1.6.35
+	github.com/darkweak/souin v1.6.36
 )
 
 require (
@@ -86,4 +86,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/darkweak/souin v1.6.35 => ../..
+replace github.com/darkweak/souin v1.6.36 => ../..
diff --git a/plugins/caddy/app.go b/plugins/caddy/app.go
@@ -17,7 +17,7 @@ type SouinApp struct {
 	// Surrogate storage to support th econfiguration reload without surrogate-key data loss.
 	SurrogateStorage providers.SurrogateInterface
 	// Cache-key tweaking.
-	CacheKeys map[string]configurationtypes.Key `json:"cache_keys,omitempty"`
+	CacheKeys configurationtypes.CacheKeys `json:"cache_keys,omitempty"`
 	// API endpoints enablers.
 	API configurationtypes.API `json:"api,omitempty"`
 	// Logger level, fallback on caddy's one when not redefined.