Issue-128 | Allow param_ in query parameters (#175)

* Issue-128 | Allow param_ in query parameters

* created func to hash map

* using in newParamsRegistry

* using in calcQueryParamsHash

* go mod tidy changes

* Change nil and empty to return 0 for Key with default value

* Fix golang-lint

* Added tests for calcQueryParamsHash

cache/key.go

@@ -45,10 +45,13 @@ type Key struct {
 
 	// Version represents data encoding version number
 	Version int
+
+	// QueryParamsHash must contain hashed value of query params
+	QueryParamsHash uint32
 }
 
 // NewKey construct cache key from provided parameters with default version number
-func NewKey(query []byte, originParams url.Values, acceptEncoding string, paramsHash uint32) *Key {
+func NewKey(query []byte, originParams url.Values, acceptEncoding string, userParamsHash uint32, queryParamsHash uint32) *Key {
 	return &Key{
 		Query:                 query,
 		AcceptEncoding:        acceptEncoding,
@@ -60,8 +63,9 @@ func NewKey(query []byte, originParams url.Values, acceptEncoding string, params
 		Extremes:              originParams.Get("extremes"),
 		MaxResultRows:         originParams.Get("max_result_rows"),
 		ResultOverflowMode:    originParams.Get("result_overflow_mode"),
-		UserParamsHash:        paramsHash,
+		UserParamsHash:        userParamsHash,
 		Version:               Version,
+		QueryParamsHash:       queryParamsHash,
 	}
 }
 
@@ -71,9 +75,9 @@ func (k *Key) filePath(dir string) string {
 
 // String returns string representation of the key.
 func (k *Key) String() string {
-	s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q; MaxResultRows=%q; Extremes=%q; ResultOverflowMode=%q; UserParams=%d",
+	s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q; MaxResultRows=%q; Extremes=%q; ResultOverflowMode=%q; UserParams=%d; QueryParams=%d",
 		k.Version, k.Query, k.AcceptEncoding, k.DefaultFormat, k.Database, k.Compress, k.EnableHTTPCompression, k.Namespace,
-		k.MaxResultRows, k.Extremes, k.ResultOverflowMode, k.UserParamsHash)
+		k.MaxResultRows, k.Extremes, k.ResultOverflowMode, k.UserParamsHash, k.QueryParamsHash)
 	h := sha256.Sum256([]byte(s))
 
 	// The first 16 bytes of the hash should be enough

cache/key_test.go

@@ -15,15 +15,15 @@ func TestKeyString(t *testing.T) {
 				Query:   []byte("SELECT 1 FROM system.numbers LIMIT 10"),
 				Version: 2,
 			},
-			expected: "bebe3382e36ffdeea479b45d827b208a",
+			expected: "ef4c039ea06ce6fd95f4ffef551ba029",
 		},
 		{
 			key: &Key{
 				Query:          []byte("SELECT 1 FROM system.numbers LIMIT 10"),
 				AcceptEncoding: "gzip",
 				Version:        2,
 			},
-			expected: "498c1af30fb94280fd7c7225c0c8fb39",
+			expected: "cb83c486eea079a87a6e567ba9869111",
 		},
 		{
 			key: &Key{
@@ -32,7 +32,7 @@ func TestKeyString(t *testing.T) {
 				DefaultFormat:  "JSON",
 				Version:        2,
 			},
-			expected: "720292aa0647cc5e53e0b6e6033eef34",
+			expected: "89edc4ac678557d80063d1060b712808",
 		},
 		{
 			key: &Key{
@@ -42,7 +42,7 @@ func TestKeyString(t *testing.T) {
 				Database:       "foobar",
 				Version:        2,
 			},
-			expected: "5c6a70736d71e570faca739c4557780c",
+			expected: "120d73469183ace3a31c941cfcc8dc13",
 		},
 		{
 			key: &Key{
@@ -53,7 +53,7 @@ func TestKeyString(t *testing.T) {
 				Namespace:      "ns123",
 				Version:        2,
 			},
-			expected: "08b4baf6825e53bbd18136a88abda4f8",
+			expected: "8441149c2cba1503e201aa94cda949f7",
 		},
 		{
 			key: &Key{
@@ -65,7 +65,23 @@ func TestKeyString(t *testing.T) {
 				Namespace:      "ns123",
 				Version:        2,
 			},
-			expected: "0e043f23ccd1b9039b33623b3b7c114a",
+			expected: "882a1cfc54f86e75a3ee89757bd33672",
+		},
+		{
+			key: &Key{
+				Query:           []byte("SELECT * FROM {table_name:Identifier} LIMIT 10"),
+				QueryParamsHash: 3825709,
+				Version:         3,
+			},
+			expected: "9d7a76630ca453d120a7349c4b6fa23d",
+		},
+		{
+			key: &Key{
+				Query:           []byte("SELECT * FROM {table_name:Identifier} LIMIT 10"),
+				QueryParamsHash: 3825710,
+				Version:         3,
+			},
+			expected: "1899cf94d4c5a3dda9575df7d8734e9b",
 		},
 	}
 

config/config_test.go

@@ -178,18 +178,18 @@ var fullConfig = Config{
 
 	Users: []User{
 		{
-			Name:         "web",
-			Password:     "****",
-			ToCluster:    "first cluster",
-			ToUser:       "web",
-			DenyHTTP:     true,
-			AllowCORS:    true,
-			ReqPerMin:    4,
-			MaxQueueSize: 100,
-			MaxQueueTime: Duration(35 * time.Second),
+			Name:             "web",
+			Password:         "****",
+			ToCluster:        "first cluster",
+			ToUser:           "web",
+			DenyHTTP:         true,
+			AllowCORS:        true,
+			ReqPerMin:        4,
+			MaxQueueSize:     100,
+			MaxQueueTime:     Duration(35 * time.Second),
 			MaxExecutionTime: Duration(30 * time.Second),
-			Cache:        "longterm",
-			Params:       "web",
+			Cache:            "longterm",
+			Params:           "web",
 		},
 		{
 			Name:                 "default",
@@ -275,9 +275,9 @@ func TestLoadConfig(t *testing.T) {
 				},
 				Users: []User{
 					{
-						Name:      "default",
-						ToCluster: "cluster",
-						ToUser:    "default",
+						Name:             "default",
+						ToCluster:        "cluster",
+						ToUser:           "default",
 						MaxExecutionTime: Duration(30 * time.Second),
 					},
 				},

go.mod

@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/DataDog/zstd v1.5.0
-	github.com/alicebob/miniredis/v2 v2.18.0
+	github.com/alicebob/miniredis/v2 v2.21.0
 	github.com/go-redis/redis/v8 v8.11.4
 	github.com/google/go-cmp v0.5.7
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
@@ -28,7 +28,7 @@ require (
 	github.com/prometheus/client_model v0.1.0 // indirect
 	github.com/prometheus/common v0.7.0 // indirect
 	github.com/prometheus/procfs v0.0.8 // indirect
-	github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect
+	github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect
 	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
 	golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
 	golang.org/x/text v0.3.6 // indirect

go.sum

@@ -6,8 +6,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
 github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis/v2 v2.18.0 h1:EPUGD69ou4Uw4c81t9NLh0+dSou46k4tFEvf498FJ0g=
-github.com/alicebob/miniredis/v2 v2.18.0/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODVXVpYDRaGhWaL6I=
+github.com/alicebob/miniredis/v2 v2.21.0 h1:CdmwIlKUWFBDS+4464GtQiQ0R1vpzOgu4Vnd74rBL7M=
+github.com/alicebob/miniredis/v2 v2.21.0/go.mod h1:XNqvJdQJv5mSuVMc0ynneafpnL/zv52acZ6kqeS0t88=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -119,8 +119,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg=
-github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
+github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 h1:k/gmLsJDWwWqbLCur2yWnJzwQEKRcAHXo6seXGuSwWw=
+github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

proxy.go

@@ -10,6 +10,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -254,11 +255,14 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 		"cluster_user": s.labels["cluster_user"],
 	}
 
-	var paramsHash uint32
+	var userParamsHash uint32
 	if s.user.params != nil {
-		paramsHash = s.user.params.key
+		userParamsHash = s.user.params.key
 	}
-	key := cache.NewKey(skipLeadingComments(q), origParams, sortHeader(req.Header.Get("Accept-Encoding")), paramsHash)
+
+	queryParamsHash := calcQueryParamsHash(origParams)
+
+	key := cache.NewKey(skipLeadingComments(q), origParams, sortHeader(req.Header.Get("Accept-Encoding")), userParamsHash, queryParamsHash)
 
 	startTime := time.Now()
 	userCache := s.user.cache
@@ -358,6 +362,21 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 	}
 }
 
+func calcQueryParamsHash(origParams url.Values) uint32 {
+	queryParams := make(map[string]string)
+	for param := range origParams {
+		if strings.HasPrefix(param, "param_") {
+			queryParams[param] = origParams.Get(param)
+		}
+	}
+	var queryParamsHash, err = calcMapHash(queryParams)
+	if err != nil {
+		log.Errorf("fail to calc hash for params %s; %s", origParams, err)
+		return 0
+	}
+	return queryParamsHash
+}
+
 // applyConfig applies the given cfg to reverseProxy.
 //
 // New config is applied only if non-nil error returned.

proxy_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"fmt"
+	"github.com/stretchr/testify/assert"
 	"io"
 	"io/ioutil"
 	"math/rand"
@@ -761,3 +762,59 @@ func (r *requestRegistry) get(key string) (bool, error) {
 	}
 	return v, nil
 }
+
+func TestCalcQueryParamsHash(t *testing.T) {
+	testCases := []struct {
+		name           string
+		input          url.Values
+		expectedResult uint32
+	}{
+		{
+			"nil Value",
+			nil,
+			0,
+		},
+		{
+			"empty calcQueryParamsHash",
+			url.Values{},
+			0,
+		},
+		{
+			"map with non param_ value",
+			url.Values{"session_id": {"foo", "bar"}},
+			0,
+		},
+		{
+			"map with only param_ value",
+			url.Values{"param_limit": {"1"}},
+			0x94a386,
+		},
+		{
+			"map with only param_ value. value affects result",
+			url.Values{"param_limit": {"2"}},
+			0x329bae01,
+		},
+		{
+			"map with mix of param_ and non-param_ value",
+			url.Values{"param_limit": {"1"}, "session_id": {"foo", "bar"}},
+			0x94a386,
+		},
+		{
+			"map with multiple param_ values",
+			url.Values{"param_limit": {"1", "2"}, "param_table": {"foo"}},
+			0x3a8a5c31,
+		},
+		{
+			"map with multiple param_ values and only first value in array affects result",
+			url.Values{"param_limit": {"1"}, "param_table": {"foo", "bar"}},
+			0x3a8a5c31,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			r := calcQueryParamsHash(tc.input)
+			assert.Equal(t, r, tc.expectedResult)
+		})
+	}
+}

scope.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"hash/fnv"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -343,6 +342,13 @@ func (s *scope) decorateRequest(req *http.Request) (*http.Request, url.Values) {
 		}
 	}
 
+	// Keep parametrized queries params
+	for param := range origParams {
+		if strings.HasPrefix(param, "param_") {
+			params.Set(param, origParams.Get(param))
+		}
+	}
+
 	// Keep external_data params
 	if req.Method == "POST" {
 		ct := req.Header.Get("Content-Type")
@@ -426,13 +432,18 @@ func newParamsRegistry(params []config.Param) (*paramsRegistry, error) {
 	if len(params) == 0 {
 		return nil, fmt.Errorf("params can't be empty")
 	}
-	h := fnv.New32a()
-	for _, p := range params {
-		str := fmt.Sprintf("%s=%s&", p.Key, p.Value)
-		h.Write([]byte(str))
+
+	var paramsMap map[string]string
+	for _, k := range params {
+		paramsMap[k.Key] = k.Value
 	}
+	key, err := calcMapHash(paramsMap)
+	if err != nil {
+		return nil, err
+	}
+
 	return &paramsRegistry{
-		key:    h.Sum32(),
+		key:    key,
 		params: params,
 	}, nil
 }

utils.go

@@ -268,3 +268,18 @@ func (dc chDecompressor) decompress(r io.Reader) ([]byte, error) {
 	lr := chdecompressor.NewReader(r)
 	return ioutil.ReadAll(lr)
 }
+
+func calcMapHash(m map[string]string) (uint32, error) {
+	if len(m) == 0 {
+		return 0, nil
+	}
+	h := fnv.New32a()
+	for k, v := range m {
+		str := fmt.Sprintf("%s=%s&", k, v)
+		_, err := h.Write([]byte(str))
+		if err != nil {
+			return 0, err
+		}
+	}
+	return h.Sum32(), nil
+}