From 02bf2737fc8a0eb43b1a64cbfa11dc47b86440d0 Mon Sep 17 00:00:00 2001 From: Massimiliano Ziccardi Date: Thu, 6 Jun 2024 17:07:58 +0200 Subject: [PATCH] add jsonb support --- go.mod | 14 +++++------ go.sum | 19 +++++++++++++++ pkg/services/queryparser/query_parser.go | 16 ++++++++++++- pkg/services/queryparser/query_parser_test.go | 8 +++++++ .../utils/stringscanner/sql_string_scanner.go | 2 ++ .../stringscanner/sql_string_scanner_test.go | 24 +++++++++++++++++++ 6 files changed, 75 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f64e49dda..5804d1b29 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.3 // indirect github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103 github.com/olekukonko/tablewriter v0.0.5 - github.com/onsi/gomega v1.27.6 + github.com/onsi/gomega v1.27.8 github.com/openshift-online/ocm-sdk-go v0.1.331 github.com/openshift/api v3.9.0+incompatible github.com/operator-framework/api v0.17.3 @@ -87,7 +87,7 @@ require ( github.com/docker/distribution v2.8.2+incompatible // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.19.14 // indirect @@ -143,13 +143,13 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 golang.org/x/crypto v0.7.0 // indirect - golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.9.1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.29.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 9d040c8e6..bc50bd409 100644 --- a/go.sum +++ b/go.sum @@ -166,10 +166,13 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -192,6 +195,7 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/goava/di v1.11.1 h1:9NBVyaoa0A5fmAfwWEaA8odHGWdgXTLW4EOti4qo72U= github.com/goava/di v1.11.1/go.mod h1:ToepvYlpTdC7DrFggmv/TyKIuezBLvAXlRxJkOvtemo= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -497,12 +501,16 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= +github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/openshift-online/ocm-sdk-go v0.1.331 h1:itr1qCJja8Edrjezxoq7FPG4NBAllWNyg4Ahxcn1GVE= github.com/openshift-online/ocm-sdk-go v0.1.331/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= @@ -699,6 +707,8 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -751,6 +761,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -774,6 +786,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -837,12 +850,16 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -915,6 +932,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/services/queryparser/query_parser.go b/pkg/services/queryparser/query_parser.go index c35378c83..08fcbd619 100644 --- a/pkg/services/queryparser/query_parser.go +++ b/pkg/services/queryparser/query_parser.go @@ -40,6 +40,12 @@ const ( and = "AND" or = "OR" not = "NOT" + + jsonbFamily = "JSONB" + jsonbField = "JSON_FIELD" + jsonbArrow = "JSONB_ARROW" + jsonbToString = "->>" + jsonbFieldToStringify = "JSONB_FIELD_TO_STRINGIFY" ) const MaximumComplexity = 10 @@ -183,11 +189,15 @@ func (p *queryParser) initStateMachine() (*state_machine.State, checkUnbalancedB {Name: and, Family: logicalOpTokenFamily, AcceptPattern: `[Aa][Nn][Dd]`}, {Name: or, Family: logicalOpTokenFamily, AcceptPattern: `[Oo][Rr]`}, {Name: not, Family: logicalOpTokenFamily, AcceptPattern: `[Nn][Oo][Tt]`}, + {Name: jsonbArrow, Family: jsonbFamily, AcceptPattern: `->`}, + {Name: jsonbField, Family: jsonbFamily, AcceptPattern: `'([^']|\\')*'`}, + {Name: jsonbToString, Family: jsonbFamily, AcceptPattern: `->>`}, + {Name: jsonbFieldToStringify, Family: jsonbFamily, AcceptPattern: `'([^']|\\')*'`}, }, Transitions: []state_machine.TokenTransitions{ {TokenName: state_machine.StartState, ValidTransitions: []string{column, openBrace}}, {TokenName: openBrace, ValidTransitions: []string{column, openBrace}}, - {TokenName: column, ValidTransitions: []string{eq, notEq, like, ilike, in, not}}, + {TokenName: column, ValidTransitions: []string{eq, notEq, like, ilike, in, not, jsonbArrow}}, {TokenName: eq, ValidTransitions: []string{quotedValue, value}}, {TokenName: notEq, ValidTransitions: []string{quotedValue, value}}, {TokenName: like, ValidTransitions: []string{quotedValue, value}}, @@ -203,6 +213,10 @@ func (p *queryParser) initStateMachine() (*state_machine.State, checkUnbalancedB {TokenName: quotedValueInList, ValidTransitions: []string{comma, closedBrace}}, {TokenName: valueInList, ValidTransitions: []string{comma, closedBrace}}, {TokenName: comma, ValidTransitions: []string{quotedValueInList, valueInList}}, + {TokenName: jsonbArrow, ValidTransitions: []string{jsonbField}}, + {TokenName: jsonbField, ValidTransitions: []string{jsonbArrow, jsonbToString}}, + {TokenName: jsonbToString, ValidTransitions: []string{jsonbFieldToStringify}}, + {TokenName: jsonbFieldToStringify, ValidTransitions: []string{eq, notEq, like, ilike, in, not}}, }, } diff --git a/pkg/services/queryparser/query_parser_test.go b/pkg/services/queryparser/query_parser_test.go index fe3f6b4d0..962554836 100644 --- a/pkg/services/queryparser/query_parser_test.go +++ b/pkg/services/queryparser/query_parser_test.go @@ -159,6 +159,14 @@ func Test_QueryParser(t *testing.T) { outValues: []interface{}{"Value", "", " value2 ", "", "c", "e", "%test%"}, wantErr: false, }, + { + name: "JSONB query", + qry: `manifest->'data'->'manifest'->'metadata'->'labels'->>'foo' = 'bar'`, + qryParser: NewQueryParser("manifest"), + outQry: "manifest -> 'data' -> 'manifest' -> 'metadata' -> 'labels' ->> 'foo' = ?", + outValues: []interface{}{"bar"}, + wantErr: false, + }, { name: "10 JOINS (maximum allowed)", qry: "name = value1 " + diff --git a/pkg/shared/utils/stringscanner/sql_string_scanner.go b/pkg/shared/utils/stringscanner/sql_string_scanner.go index 603562736..0487efa35 100644 --- a/pkg/shared/utils/stringscanner/sql_string_scanner.go +++ b/pkg/shared/utils/stringscanner/sql_string_scanner.go @@ -60,6 +60,8 @@ func (s *scanner) Init(txt string) { // found closebrace Token sendCurrentTokens() s.tokens = append(s.tokens, Token{TokenType: BRACE, Value: string(currentChar), Position: i}) + case '-': + fallthrough case '=': fallthrough case '<': diff --git a/pkg/shared/utils/stringscanner/sql_string_scanner_test.go b/pkg/shared/utils/stringscanner/sql_string_scanner_test.go index 27685fd2c..abf2173f5 100644 --- a/pkg/shared/utils/stringscanner/sql_string_scanner_test.go +++ b/pkg/shared/utils/stringscanner/sql_string_scanner_test.go @@ -81,6 +81,30 @@ func Test_SQLStringScanner(t *testing.T) { {TokenType: LITERAL, Value: "3", Position: 61}, }, }, + { + name: "Test SQL with JSONB", + value: `select * from table where manifest->'data'->'manifest'->'metadata'->'labels'->>'foo' = 'bar'`, + expectedTokens: []Token{ + {TokenType: LITERAL, Value: "select", Position: 0}, + {TokenType: LITERAL, Value: "*", Position: 7}, + {TokenType: LITERAL, Value: "from", Position: 9}, + {TokenType: LITERAL, Value: "table", Position: 14}, + {TokenType: LITERAL, Value: "where", Position: 20}, + {TokenType: LITERAL, Value: "manifest", Position: 26}, + {TokenType: OP, Value: "->", Position: 34}, + {TokenType: LITERAL, Value: "'data'", Position: 36}, + {TokenType: OP, Value: "->", Position: 42}, + {TokenType: LITERAL, Value: "'manifest'", Position: 44}, + {TokenType: OP, Value: "->", Position: 54}, + {TokenType: LITERAL, Value: "'metadata'", Position: 56}, + {TokenType: OP, Value: "->", Position: 66}, + {TokenType: LITERAL, Value: "'labels'", Position: 68}, + {TokenType: OP, Value: "->>", Position: 76}, + {TokenType: LITERAL, Value: "'foo'", Position: 79}, + {TokenType: OP, Value: "=", Position: 85}, + {TokenType: LITERAL, Value: "'bar'", Position: 87}, + }, + }, } for _, testcase := range tests { tt := testcase