VReplication: Support binlog_row_value_options=PARTIAL_JSON

.github/workflows/cluster_endtoend_onlineddl_vrepl.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_schemadiff_vrepl.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_basic.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_cellalias.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_foreign_key_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_migrate.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_and_materialize | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_v2.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_vtctldclient_vdiff2_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/codecov.yml

@@ -111,7 +111,7 @@ jobs:
 
     - name: Upload coverage reports to codecov.io
       if: steps.changes.outputs.changed_files == 'true'
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # https://github.com/codecov/codecov-action/releases/tag/v5.0.7
       with:
         fail_ci_if_error: true
         verbose: true

.github/workflows/unit_test_evalengine_mysql57.yml

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_evalengine_mysql80.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_evalengine_mysql84.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql57.yml

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql80.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql84.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

go/mysql/binlog/binlog_json.go

@@ -17,6 +17,7 @@ limitations under the License.
 package binlog
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"math"
@@ -25,9 +26,12 @@ import (
 	"vitess.io/vitess/go/hack"
 	"vitess.io/vitess/go/mysql/format"
 	"vitess.io/vitess/go/mysql/json"
+	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/sqlparser"
+	"vitess.io/vitess/go/vt/vterrors"
+
 	querypb "vitess.io/vitess/go/vt/proto/query"
 	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
-	"vitess.io/vitess/go/vt/vterrors"
 )
 
 /*
@@ -44,6 +48,14 @@ https://github.com/shyiko/mysql-binlog-connector-java/pull/119/files
 https://github.com/noplay/python-mysql-replication/blob/175df28cc8b536a68522ff9b09dc5440adad6094/pymysqlreplication/packet.py
 */
 
+type jsonDiffOp uint8
+
+const (
+	jsonDiffOpReplace = jsonDiffOp(0)
+	jsonDiffOpInsert  = jsonDiffOp(1)
+	jsonDiffOpRemove  = jsonDiffOp(2)
+)
+
 // ParseBinaryJSON provides the parsing function from the mysql binary json
 // representation to a JSON value instance.
 func ParseBinaryJSON(data []byte) (*json.Value, error) {
@@ -60,6 +72,84 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) {
 	return node, nil
 }
 
+// ParseBinaryJSONDiff provides the parsing function from the binary MySQL
+// JSON diff representation to an SQL expression.
+func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) {
+	diff := bytes.Buffer{}
+	// Reasonable estimate of the space we'll need to build the SQL
+	// expression in order to try and avoid reallocations w/o
+	// overallocating too much.
+	diff.Grow(int(float32(len(data)) * 1.25))
+	pos := 0
+	outer := false
+	innerStr := ""
+
+	for pos < len(data) {
+		opType := jsonDiffOp(data[pos])
+		pos++
+		if outer {
+			innerStr = diff.String()
+			diff.Reset()
+		}
+		switch opType {
+		case jsonDiffOpReplace:
+			diff.WriteString("JSON_REPLACE(")
+		case jsonDiffOpInsert:
+			diff.WriteString("JSON_INSERT(")
+		case jsonDiffOpRemove:
+			diff.WriteString("JSON_REMOVE(")
+		default:
+			// Can be a JSON null.
+			js, err := ParseBinaryJSON(data)
+			if err == nil && js.Type() == json.TypeNull {
+				return sqltypes.MakeTrusted(sqltypes.Expression, js.MarshalTo(nil)), nil
+			}
+			return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT,
+				"invalid JSON diff operation: %d", opType)
+		}
+		if outer {
+			diff.WriteString(innerStr)
+			diff.WriteString(", ")
+		} else { // Only the inner most function has the field name
+			diff.WriteString("%s, ") // This will later be replaced by the field name
+		}
+		outer = true
+
+		pathLen, readTo := readVariableLength(data, pos)
+		pos = readTo
+		path := data[pos : pos+pathLen]
+		pos += pathLen
+		// We have to specify the unicode character set for the strings we
+		// use in the expression as the connection can be using a different
+		// character set (e.g. vreplication always uses set names binary).
+		diff.WriteString(sqlparser.Utf8mb4Str)
+		diff.WriteByte('\'')
+		diff.Write(path)
+		diff.WriteByte('\'')
+		if opType == jsonDiffOpRemove { // No value for remove
+			diff.WriteByte(')')
+			continue
+		}
+
+		diff.WriteString(", ")
+		valueLen, readTo := readVariableLength(data, pos)
+		pos = readTo
+		value, err := ParseBinaryJSON(data[pos : pos+valueLen])
+		if err != nil {
+			return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT,
+				"cannot read JSON diff value for path %s: %v", path, err)
+		}
+		pos += valueLen
+		if value.Type() == json.TypeString {
+			diff.WriteString(sqlparser.Utf8mb4Str)
+		}
+		diff.Write(value.MarshalTo(nil))
+		diff.WriteByte(')')
+	}
+
+	return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil
+}
+
 // jsonDataType has the values used in the mysql json binary representation to denote types.
 // We have string, literal(true/false/null), number, object or array types.
 // large object => doc size > 64K: you get pointers instead of inline values.
@@ -315,7 +405,7 @@ func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, er
 		precision := decimalData[0]
 		scale := decimalData[1]
 		metadata := (uint16(precision) << 8) + uint16(scale)
-		val, _, err := CellValue(decimalData, 2, TypeNewDecimal, metadata, &querypb.Field{Type: querypb.Type_DECIMAL})
+		val, _, err := CellValue(decimalData, 2, TypeNewDecimal, metadata, &querypb.Field{Type: querypb.Type_DECIMAL}, false)
 		if err != nil {
 			return nil, err
 		}