From 3e3ab1d811989ce9d3fcf0d73642729680a17fd1 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 23 Dec 2024 14:22:32 -0500 Subject: [PATCH] Run all unit tests with binlog_row_value_options=PARTIAL_JSON This way we're testing this feature both with binlog-row-image=FULL and with binlog-row-image=NOBLOB. Signed-off-by: Matt Lord --- .../vreplication/framework_test.go | 13 +- .../vreplication/vplayer_flaky_test.go | 183 ++++++++++++------ 2 files changed, 127 insertions(+), 69 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index c9c6451bbeb..9f2647a8770 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -169,8 +169,7 @@ func setup(ctx context.Context) (func(), int) { var ( // We run unit tests twice, first with binlog_row_image=FULL, then with NOBLOB. runNoBlobTest = false - // When using MySQL 8.0 or later, we also set binlog_row_value_options=PARTIAL_JSON - // when using NOBLOB. + // When using MySQL 8.0 or later, we set binlog_row_value_options=PARTIAL_JSON. runPartialJSONTest = false ) @@ -183,7 +182,11 @@ func TestMain(m *testing.M) { exitCode := func() int { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if err := utils.SetBinlogRowImageOptions("full", false, tempDir); err != nil { + // binlog-row-value-options=PARTIAL_JSON is only supported in MySQL 8.0 and later. + // We still run unit tests with MySQL 5.7, so we cannot add it to the cnf file + // when using 5.7 or mysqld will fail to start. + runPartialJSONTest = utils.CIDBPlatformIsMySQL8orLater() + if err := utils.SetBinlogRowImageOptions("full", runPartialJSONTest, tempDir); err != nil { panic(err) } defer utils.SetBinlogRowImageOptions("", false, tempDir) @@ -198,10 +201,6 @@ func TestMain(m *testing.M) { cancel() runNoBlobTest = true - // binlog-row-value-options=PARTIAL_JSON is only supported in MySQL 8.0 and later. - // We still run unit tests with MySQL 5.7, so we cannot add it to the cnf file - // when using 5.7 or mysqld will fail to start. - runPartialJSONTest = utils.CIDBPlatformIsMySQL8orLater() if err := utils.SetBinlogRowImageOptions("noblob", runPartialJSONTest, tempDir); err != nil { panic(err) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index e697f6d2752..fe96e47281c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1622,80 +1622,139 @@ func TestPlayerPartialImages(t *testing.T) { {"12", "{\"key2\": \"val2\"}", "newest blob data"}, }, }, - { - input: `update src set jd=JSON_SET(jd, '$.years', 5) where id = 1`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.years', CAST(5 as JSON)) where id=1", + }...) + if runNoBlobTest { + testCases = append(testCases, []testCase{ + { + input: `update src set jd=JSON_SET(jd, '$.years', 5) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.years', CAST(5 as JSON)) where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.hobbies', JSON_ARRAY('skiing', 'video games', 'hiking')) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.hobbies', JSON_ARRAY(_utf8mb4'skiing', _utf8mb4'video games', _utf8mb4'hiking')) where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.hobbies', JSON_ARRAY('skiing', 'video games', 'hiking')) where id = 1`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.hobbies', JSON_ARRAY(_utf8mb4'skiing', _utf8mb4'video games', _utf8mb4'hiking')) where id=1", + { + input: `update src set jd=JSON_SET(jd, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON)) where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.current', true) where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.current', CAST(_utf8mb4'true' as JSON)) where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') where id = 12`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON)) where id=12", + { + input: `update src set jd=JSON_SET(jd, '$.idontknow', null, '$.idontknoweither', 'null') where id = 3`, + output: []string{ + "update dst set jd=JSON_INSERT(JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)), _utf8mb4'$.idontknoweither', CAST(JSON_QUOTE(_utf8mb4'null') as JSON)) where id=3", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\"}", "newest blob data"}, + { + input: `update src set id = id+10 where id = 3`, + error: "binary log event missing a needed value for dst.bd due to not using binlog-row-image=FULL", }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.current', true) where id = 12`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.current', CAST(_utf8mb4'true' as JSON)) where id=12", + }...) + } else { + testCases = append(testCases, []testCase{ + { + input: `update src set jd=JSON_SET(jd, '$.years', 5) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.years', CAST(5 as JSON)), bd=_binary'blob data' where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.hobbies', JSON_ARRAY('skiing', 'video games', 'hiking')) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.hobbies', JSON_ARRAY(_utf8mb4'skiing', _utf8mb4'video games', _utf8mb4'hiking')), bd=_binary'blob data' where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.idontknow', null, '$.idontknoweither', 'null') where id = 3`, - output: []string{ - "update dst set jd=JSON_INSERT(JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)), _utf8mb4'$.idontknoweither', CAST(JSON_QUOTE(_utf8mb4'null') as JSON)) where id=3", + { + input: `update src set jd=JSON_SET(jd, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON)), bd=_binary'newest blob data' where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.current', true) where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.current', CAST(_utf8mb4'true' as JSON)), bd=_binary'newest blob data' where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - }, - }...) - if runNoBlobTest { - testCases = append(testCases, testCase{ - input: `update src set id = id+10 where id = 3`, - error: "binary log event missing a needed value for dst.bd due to not using binlog-row-image=FULL", - }) - } else { - testCases = append(testCases, testCase{ - input: `update src set id = id+10 where id = 3`, - output: []string{ - "delete from dst where id=3", - "insert into dst(id,jd,bd) values (13,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')", + { + input: `update src set jd=JSON_SET(jd, '$.idontknow', null, '$.idontknoweither', 'null') where id = 3`, + output: []string{ + "update dst set jd=JSON_INSERT(JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)), _utf8mb4'$.idontknoweither', CAST(JSON_QUOTE(_utf8mb4'null') as JSON)), bd=_binary'blob data3' where id=3", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, - {"13", "{\"key3\": \"val3\", \"idontknow\": null}", "blob data3"}, + { + input: `update src set id = id+10 where id = 3`, + output: []string{ + "delete from dst where id=3", + "insert into dst(id,jd,bd) values (13,JSON_OBJECT(_utf8mb4'idontknow', null, _utf8mb4'idontknoweither', _utf8mb4'null', _utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + {"13", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, + }, }, - }) + }...) } for _, tc := range testCases {